2024-12-09 10:15:19,085 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f 2024-12-09 10:15:19,114 main DEBUG Took 0.024901 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 10:15:19,114 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 10:15:19,114 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 10:15:19,116 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 10:15:19,170 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:15:19,196 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 10:15:19,238 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,241 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:15:19,242 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,243 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:15:19,247 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,248 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:15:19,249 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,250 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:15:19,250 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,251 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:15:19,252 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,253 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:15:19,255 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,255 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:15:19,256 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,257 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:15:19,257 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,258 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:15:19,259 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,259 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:15:19,260 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,260 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:15:19,261 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:15:19,263 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,264 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 10:15:19,266 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:15:19,268 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 10:15:19,272 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 10:15:19,273 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 10:15:19,275 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 10:15:19,276 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 10:15:19,305 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 10:15:19,308 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 10:15:19,311 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 10:15:19,312 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 10:15:19,313 main DEBUG createAppenders(={Console}) 2024-12-09 10:15:19,314 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f initialized 2024-12-09 10:15:19,315 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f 2024-12-09 10:15:19,315 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f OK. 2024-12-09 10:15:19,316 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 10:15:19,317 main DEBUG OutputStream closed 2024-12-09 10:15:19,318 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 10:15:19,318 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 10:15:19,319 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@505fc5a4 OK 2024-12-09 10:15:19,448 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 10:15:19,452 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 10:15:19,459 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 10:15:19,461 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 10:15:19,468 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 10:15:19,468 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 10:15:19,470 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 10:15:19,471 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 10:15:19,472 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 10:15:19,472 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 10:15:19,473 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 10:15:19,473 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 10:15:19,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 10:15:19,475 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 10:15:19,476 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 10:15:19,477 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 10:15:19,478 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 10:15:19,479 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 10:15:19,487 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 10:15:19,487 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@39de3d36) with optional ClassLoader: null 2024-12-09 10:15:19,488 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 10:15:19,494 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@39de3d36] started OK. 2024-12-09T10:15:19,947 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540 2024-12-09 10:15:19,952 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 10:15:19,952 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T10:15:19,992 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi timeout: 26 mins 2024-12-09T10:15:20,035 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T10:15:20,040 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/cluster_4c6bf60c-533f-d07c-4add-3b892b5aceba, deleteOnExit=true 2024-12-09T10:15:20,041 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-09T10:15:20,042 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/test.cache.data in system properties and HBase conf 2024-12-09T10:15:20,042 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T10:15:20,043 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/hadoop.log.dir in system properties and HBase conf 2024-12-09T10:15:20,044 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T10:15:20,045 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T10:15:20,046 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-09T10:15:20,236 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T10:15:20,375 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T10:15:20,383 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T10:15:20,384 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T10:15:20,384 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T10:15:20,385 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T10:15:20,386 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T10:15:20,386 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T10:15:20,387 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T10:15:20,387 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T10:15:20,388 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T10:15:20,388 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/nfs.dump.dir in system properties and HBase conf 2024-12-09T10:15:20,389 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/java.io.tmpdir in system properties and HBase conf 2024-12-09T10:15:20,389 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T10:15:20,390 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T10:15:20,390 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T10:15:21,858 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T10:15:21,978 INFO [Time-limited test {}] log.Log(170): Logging initialized @4080ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T10:15:22,097 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T10:15:22,203 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T10:15:22,236 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T10:15:22,237 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T10:15:22,239 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T10:15:22,266 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T10:15:22,274 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@842d13f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/hadoop.log.dir/,AVAILABLE} 2024-12-09T10:15:22,276 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@201ce359{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T10:15:22,554 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2490b214{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/java.io.tmpdir/jetty-localhost-44701-hadoop-hdfs-3_4_1-tests_jar-_-any-8162298484987001573/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T10:15:22,576 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1f7e2804{HTTP/1.1, (http/1.1)}{localhost:44701} 2024-12-09T10:15:22,577 INFO [Time-limited test {}] server.Server(415): Started @4680ms 2024-12-09T10:15:23,317 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T10:15:23,326 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T10:15:23,332 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T10:15:23,332 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T10:15:23,333 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T10:15:23,334 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5854846b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/hadoop.log.dir/,AVAILABLE} 2024-12-09T10:15:23,336 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a2745b0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T10:15:23,475 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7d75efc4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/java.io.tmpdir/jetty-localhost-34577-hadoop-hdfs-3_4_1-tests_jar-_-any-17972955009101671762/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T10:15:23,476 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@723ec49{HTTP/1.1, (http/1.1)}{localhost:34577} 2024-12-09T10:15:23,477 INFO [Time-limited test {}] server.Server(415): Started @5580ms 2024-12-09T10:15:23,554 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T10:15:24,710 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/cluster_4c6bf60c-533f-d07c-4add-3b892b5aceba/dfs/data/data1/current/BP-1908975505-172.17.0.3-1733739321312/current, will proceed with Du for space computation calculation, 2024-12-09T10:15:24,711 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/cluster_4c6bf60c-533f-d07c-4add-3b892b5aceba/dfs/data/data2/current/BP-1908975505-172.17.0.3-1733739321312/current, will proceed with Du for space computation calculation, 2024-12-09T10:15:24,776 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T10:15:24,851 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x990f362a70e8e570 with lease ID 0x33c3f01efa46cf5: Processing first storage report for DS-99aab850-549c-4f09-862a-e7fc3422e5a2 from datanode DatanodeRegistration(127.0.0.1:34711, datanodeUuid=197ca379-f6d4-4524-baa8-d926b31d84e5, infoPort=44799, infoSecurePort=0, ipcPort=33893, storageInfo=lv=-57;cid=testClusterID;nsid=1426845827;c=1733739321312) 2024-12-09T10:15:24,853 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x990f362a70e8e570 with lease ID 0x33c3f01efa46cf5: from storage DS-99aab850-549c-4f09-862a-e7fc3422e5a2 node DatanodeRegistration(127.0.0.1:34711, datanodeUuid=197ca379-f6d4-4524-baa8-d926b31d84e5, infoPort=44799, infoSecurePort=0, ipcPort=33893, storageInfo=lv=-57;cid=testClusterID;nsid=1426845827;c=1733739321312), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T10:15:24,854 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x990f362a70e8e570 with lease ID 0x33c3f01efa46cf5: Processing first storage report for DS-c4770230-27a9-4eea-9d76-8ec26c4c9eb5 from datanode DatanodeRegistration(127.0.0.1:34711, datanodeUuid=197ca379-f6d4-4524-baa8-d926b31d84e5, infoPort=44799, infoSecurePort=0, ipcPort=33893, storageInfo=lv=-57;cid=testClusterID;nsid=1426845827;c=1733739321312) 2024-12-09T10:15:24,855 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x990f362a70e8e570 with lease ID 0x33c3f01efa46cf5: from storage DS-c4770230-27a9-4eea-9d76-8ec26c4c9eb5 node DatanodeRegistration(127.0.0.1:34711, datanodeUuid=197ca379-f6d4-4524-baa8-d926b31d84e5, infoPort=44799, infoSecurePort=0, ipcPort=33893, storageInfo=lv=-57;cid=testClusterID;nsid=1426845827;c=1733739321312), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T10:15:24,924 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540 2024-12-09T10:15:25,057 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/cluster_4c6bf60c-533f-d07c-4add-3b892b5aceba/zookeeper_0, clientPort=64103, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/cluster_4c6bf60c-533f-d07c-4add-3b892b5aceba/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/cluster_4c6bf60c-533f-d07c-4add-3b892b5aceba/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T10:15:25,071 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=64103 2024-12-09T10:15:25,085 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:15:25,089 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:15:25,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741825_1001 (size=7) 2024-12-09T10:15:25,856 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc with version=8 2024-12-09T10:15:25,857 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/hbase-staging 2024-12-09T10:15:26,026 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T10:15:26,327 INFO [Time-limited test {}] client.ConnectionUtils(129): master/de59e52b2fff:0 server-side Connection retries=6 2024-12-09T10:15:26,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T10:15:26,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T10:15:26,352 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T10:15:26,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T10:15:26,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T10:15:26,511 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T10:15:26,577 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T10:15:26,589 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T10:15:26,594 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T10:15:26,625 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 25030 (auto-detected) 2024-12-09T10:15:26,627 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-09T10:15:26,650 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.3:37695 2024-12-09T10:15:26,659 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:15:26,661 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:15:26,674 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:37695 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:26,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:376950x0, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:26,779 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37695-0x100be8885930000 connected 2024-12-09T10:15:26,958 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T10:15:26,964 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T10:15:26,971 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T10:15:26,982 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37695 2024-12-09T10:15:26,983 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37695 2024-12-09T10:15:26,985 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37695 2024-12-09T10:15:26,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37695 2024-12-09T10:15:26,991 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37695 2024-12-09T10:15:27,002 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc, hbase.cluster.distributed=false 2024-12-09T10:15:27,103 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/de59e52b2fff:0 server-side Connection retries=6 2024-12-09T10:15:27,104 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T10:15:27,104 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T10:15:27,104 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T10:15:27,104 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T10:15:27,105 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T10:15:27,107 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T10:15:27,113 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T10:15:27,114 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.3:37621 2024-12-09T10:15:27,116 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T10:15:27,123 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T10:15:27,125 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:15:27,128 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:15:27,133 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:37621 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:27,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376210x0, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:27,142 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:376210x0, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T10:15:27,142 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37621-0x100be8885930001 connected 2024-12-09T10:15:27,144 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T10:15:27,145 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T10:15:27,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37621 2024-12-09T10:15:27,151 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37621 2024-12-09T10:15:27,159 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37621 2024-12-09T10:15:27,160 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37621 2024-12-09T10:15:27,161 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37621 2024-12-09T10:15:27,166 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/de59e52b2fff,37695,1733739326017 2024-12-09T10:15:27,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T10:15:27,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T10:15:27,187 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/de59e52b2fff,37695,1733739326017 2024-12-09T10:15:27,188 DEBUG [M:0;de59e52b2fff:37695 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;de59e52b2fff:37695 2024-12-09T10:15:27,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T10:15:27,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T10:15:27,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:27,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:27,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T10:15:27,232 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T10:15:27,233 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/de59e52b2fff,37695,1733739326017 from backup master directory 2024-12-09T10:15:27,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/de59e52b2fff,37695,1733739326017 2024-12-09T10:15:27,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T10:15:27,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T10:15:27,244 WARN [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T10:15:27,244 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=de59e52b2fff,37695,1733739326017 2024-12-09T10:15:27,247 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T10:15:27,251 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T10:15:27,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741826_1002 (size=42) 2024-12-09T10:15:27,754 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/hbase.id with ID: 0c6f32a7-e3f8-4a47-90f8-ff1b85bfb52d 2024-12-09T10:15:27,824 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:15:27,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:27,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:27,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741827_1003 (size=196) 2024-12-09T10:15:28,360 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T10:15:28,364 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T10:15:28,399 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:28,405 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T10:15:28,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741828_1004 (size=1189) 2024-12-09T10:15:28,872 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store 2024-12-09T10:15:28,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741829_1005 (size=34) 2024-12-09T10:15:29,322 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T10:15:29,322 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:15:29,324 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T10:15:29,324 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:15:29,325 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:15:29,325 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T10:15:29,325 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:15:29,325 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:15:29,326 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-09T10:15:29,329 WARN [master/de59e52b2fff:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/.initializing 2024-12-09T10:15:29,329 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/WALs/de59e52b2fff,37695,1733739326017 2024-12-09T10:15:29,340 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T10:15:29,357 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=de59e52b2fff%2C37695%2C1733739326017, suffix=, logDir=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/WALs/de59e52b2fff,37695,1733739326017, archiveDir=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/oldWALs, maxLogs=10 2024-12-09T10:15:29,384 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(617): When create output stream for /user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/WALs/de59e52b2fff,37695,1733739326017/de59e52b2fff%2C37695%2C1733739326017.1733739329364, exclude list is [], retry=0 2024-12-09T10:15:29,404 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34711,DS-99aab850-549c-4f09-862a-e7fc3422e5a2,DISK] 2024-12-09T10:15:29,407 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T10:15:29,466 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/WALs/de59e52b2fff,37695,1733739326017/de59e52b2fff%2C37695%2C1733739326017.1733739329364 2024-12-09T10:15:29,468 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44799:44799)] 2024-12-09T10:15:29,469 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:15:29,469 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:15:29,474 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:15:29,475 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:15:29,548 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:15:29,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T10:15:29,593 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:15:29,600 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:15:29,601 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:15:29,609 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T10:15:29,610 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:15:29,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:15:29,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:15:29,621 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T10:15:29,621 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:15:29,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:15:29,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:15:29,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T10:15:29,630 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:15:29,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:15:29,640 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:15:29,642 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:15:29,659 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T10:15:29,675 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:15:29,703 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T10:15:29,711 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68988103, jitterRate=0.028002843260765076}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T10:15:29,718 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-09T10:15:29,722 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T10:15:29,780 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cbbbb4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:29,852 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-09T10:15:29,868 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T10:15:29,868 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T10:15:29,870 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T10:15:29,876 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 5 msec 2024-12-09T10:15:29,883 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 5 msec 2024-12-09T10:15:29,883 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T10:15:29,926 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T10:15:29,949 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T10:15:29,979 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-09T10:15:29,983 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T10:15:29,992 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T10:15:30,007 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-09T10:15:30,010 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T10:15:30,031 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T10:15:30,048 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-09T10:15:30,060 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T10:15:30,075 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T10:15:30,110 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T10:15:30,124 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T10:15:30,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T10:15:30,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T10:15:30,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:30,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:30,152 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=de59e52b2fff,37695,1733739326017, sessionid=0x100be8885930000, setting cluster-up flag (Was=false) 2024-12-09T10:15:30,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:30,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:30,275 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T10:15:30,303 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=de59e52b2fff,37695,1733739326017 2024-12-09T10:15:30,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:30,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:30,349 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T10:15:30,351 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=de59e52b2fff,37695,1733739326017 2024-12-09T10:15:30,404 DEBUG [RS:0;de59e52b2fff:37621 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;de59e52b2fff:37621 2024-12-09T10:15:30,427 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1008): ClusterId : 0c6f32a7-e3f8-4a47-90f8-ff1b85bfb52d 2024-12-09T10:15:30,435 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T10:15:30,480 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T10:15:30,487 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T10:15:30,516 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T10:15:30,517 DEBUG [RS:0;de59e52b2fff:37621 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73878615, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:30,519 DEBUG [RS:0;de59e52b2fff:37621 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d57e9bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=de59e52b2fff/172.17.0.3:0 2024-12-09T10:15:30,523 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-09T10:15:30,523 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-09T10:15:30,523 DEBUG [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-09T10:15:30,529 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-09T10:15:30,533 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(3073): reportForDuty to master=de59e52b2fff,37695,1733739326017 with isa=de59e52b2fff/172.17.0.3:37621, startcode=1733739327100 2024-12-09T10:15:30,537 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-09T10:15:30,542 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T10:15:30,552 DEBUG [RS:0;de59e52b2fff:37621 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T10:15:30,549 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: de59e52b2fff,37695,1733739326017 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T10:15:30,555 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/de59e52b2fff:0, corePoolSize=5, maxPoolSize=5 2024-12-09T10:15:30,555 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/de59e52b2fff:0, corePoolSize=5, maxPoolSize=5 2024-12-09T10:15:30,556 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/de59e52b2fff:0, corePoolSize=5, maxPoolSize=5 2024-12-09T10:15:30,556 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/de59e52b2fff:0, corePoolSize=5, maxPoolSize=5 2024-12-09T10:15:30,556 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/de59e52b2fff:0, corePoolSize=10, maxPoolSize=10 2024-12-09T10:15:30,556 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/de59e52b2fff:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:15:30,556 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/de59e52b2fff:0, corePoolSize=2, maxPoolSize=2 2024-12-09T10:15:30,561 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/de59e52b2fff:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:15:30,594 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-09T10:15:30,594 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-09T10:15:30,620 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733739360619 2024-12-09T10:15:30,622 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T10:15:30,641 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T10:15:30,637 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:15:30,643 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T10:15:30,653 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T10:15:30,654 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T10:15:30,654 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T10:15:30,654 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T10:15:30,656 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45735, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T10:15:30,659 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:30,670 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T10:15:30,670 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37695 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3280) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:593) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-09T10:15:30,672 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T10:15:30,672 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T10:15:30,678 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T10:15:30,679 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T10:15:30,685 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/de59e52b2fff:0:becomeActiveMaster-HFileCleaner.large.0-1733739330681,5,FailOnTimeoutGroup] 2024-12-09T10:15:30,686 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/de59e52b2fff:0:becomeActiveMaster-HFileCleaner.small.0-1733739330685,5,FailOnTimeoutGroup] 2024-12-09T10:15:30,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741831_1007 (size=1039) 2024-12-09T10:15:30,686 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:30,688 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T10:15:30,689 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:30,690 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:30,712 DEBUG [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(3097): Master is not running yet 2024-12-09T10:15:30,712 WARN [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1099): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T10:15:30,814 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(3073): reportForDuty to master=de59e52b2fff,37695,1733739326017 with isa=de59e52b2fff/172.17.0.3:37621, startcode=1733739327100 2024-12-09T10:15:30,816 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37695 {}] master.ServerManager(332): Checking decommissioned status of RegionServer de59e52b2fff,37621,1733739327100 2024-12-09T10:15:30,819 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37695 {}] master.ServerManager(486): Registering regionserver=de59e52b2fff,37621,1733739327100 2024-12-09T10:15:30,831 DEBUG [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc 2024-12-09T10:15:30,831 DEBUG [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:33961 2024-12-09T10:15:30,831 DEBUG [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-09T10:15:30,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T10:15:30,872 DEBUG [RS:0;de59e52b2fff:37621 {}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/de59e52b2fff,37621,1733739327100 2024-12-09T10:15:30,873 WARN [RS:0;de59e52b2fff:37621 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T10:15:30,873 INFO [RS:0;de59e52b2fff:37621 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T10:15:30,874 DEBUG [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/WALs/de59e52b2fff,37621,1733739327100 2024-12-09T10:15:30,875 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [de59e52b2fff,37621,1733739327100] 2024-12-09T10:15:30,926 DEBUG [RS:0;de59e52b2fff:37621 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-09T10:15:30,950 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T10:15:30,981 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T10:15:30,992 INFO [RS:0;de59e52b2fff:37621 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T10:15:30,992 INFO [RS:0;de59e52b2fff:37621 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:31,000 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-09T10:15:31,020 INFO [RS:0;de59e52b2fff:37621 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:31,021 DEBUG [RS:0;de59e52b2fff:37621 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/de59e52b2fff:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:15:31,022 DEBUG [RS:0;de59e52b2fff:37621 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/de59e52b2fff:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:15:31,023 DEBUG [RS:0;de59e52b2fff:37621 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:15:31,023 DEBUG [RS:0;de59e52b2fff:37621 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/de59e52b2fff:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:15:31,023 DEBUG [RS:0;de59e52b2fff:37621 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/de59e52b2fff:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:15:31,023 DEBUG [RS:0;de59e52b2fff:37621 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/de59e52b2fff:0, corePoolSize=2, maxPoolSize=2 2024-12-09T10:15:31,023 DEBUG [RS:0;de59e52b2fff:37621 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/de59e52b2fff:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:15:31,024 DEBUG [RS:0;de59e52b2fff:37621 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/de59e52b2fff:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:15:31,024 DEBUG [RS:0;de59e52b2fff:37621 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/de59e52b2fff:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:15:31,024 DEBUG [RS:0;de59e52b2fff:37621 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/de59e52b2fff:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:15:31,024 DEBUG [RS:0;de59e52b2fff:37621 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/de59e52b2fff:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:15:31,024 DEBUG [RS:0;de59e52b2fff:37621 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/de59e52b2fff:0, corePoolSize=3, maxPoolSize=3 2024-12-09T10:15:31,024 DEBUG [RS:0;de59e52b2fff:37621 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/de59e52b2fff:0, corePoolSize=3, maxPoolSize=3 2024-12-09T10:15:31,035 INFO [RS:0;de59e52b2fff:37621 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:31,036 INFO [RS:0;de59e52b2fff:37621 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:31,036 INFO [RS:0;de59e52b2fff:37621 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:31,036 INFO [RS:0;de59e52b2fff:37621 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:31,036 INFO [RS:0;de59e52b2fff:37621 {}] hbase.ChoreService(168): Chore ScheduledChore name=de59e52b2fff,37621,1733739327100-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T10:15:31,093 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T10:15:31,096 INFO [RS:0;de59e52b2fff:37621 {}] hbase.ChoreService(168): Chore ScheduledChore name=de59e52b2fff,37621,1733739327100-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:31,103 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-09T10:15:31,104 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc 2024-12-09T10:15:31,140 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.Replication(204): de59e52b2fff,37621,1733739327100 started 2024-12-09T10:15:31,140 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1767): Serving as de59e52b2fff,37621,1733739327100, RpcServer on de59e52b2fff/172.17.0.3:37621, sessionid=0x100be8885930001 2024-12-09T10:15:31,144 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T10:15:31,144 DEBUG [RS:0;de59e52b2fff:37621 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager de59e52b2fff,37621,1733739327100 2024-12-09T10:15:31,144 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'de59e52b2fff,37621,1733739327100' 2024-12-09T10:15:31,144 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T10:15:31,150 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T10:15:31,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741832_1008 (size=32) 2024-12-09T10:15:31,159 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T10:15:31,159 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T10:15:31,159 DEBUG [RS:0;de59e52b2fff:37621 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager de59e52b2fff,37621,1733739327100 2024-12-09T10:15:31,159 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'de59e52b2fff,37621,1733739327100' 2024-12-09T10:15:31,159 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T10:15:31,166 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T10:15:31,171 DEBUG [RS:0;de59e52b2fff:37621 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T10:15:31,171 INFO [RS:0;de59e52b2fff:37621 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T10:15:31,171 INFO [RS:0;de59e52b2fff:37621 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T10:15:31,278 INFO [RS:0;de59e52b2fff:37621 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T10:15:31,283 INFO [RS:0;de59e52b2fff:37621 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=de59e52b2fff%2C37621%2C1733739327100, suffix=, logDir=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/WALs/de59e52b2fff,37621,1733739327100, archiveDir=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/oldWALs, maxLogs=32 2024-12-09T10:15:31,301 DEBUG [RS:0;de59e52b2fff:37621 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(617): When create output stream for /user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/WALs/de59e52b2fff,37621,1733739327100/de59e52b2fff%2C37621%2C1733739327100.1733739331285, exclude list is [], retry=0 2024-12-09T10:15:31,307 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34711,DS-99aab850-549c-4f09-862a-e7fc3422e5a2,DISK] 2024-12-09T10:15:31,317 INFO [RS:0;de59e52b2fff:37621 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/WALs/de59e52b2fff,37621,1733739327100/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:15:31,318 DEBUG [RS:0;de59e52b2fff:37621 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44799:44799)] 2024-12-09T10:15:31,557 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:15:31,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T10:15:31,565 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T10:15:31,566 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:15:31,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:15:31,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T10:15:31,572 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T10:15:31,572 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:15:31,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:15:31,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T10:15:31,579 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T10:15:31,579 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:15:31,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:15:31,584 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740 2024-12-09T10:15:31,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740 2024-12-09T10:15:31,591 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T10:15:31,596 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-09T10:15:31,604 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T10:15:31,609 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62454413, jitterRate=-0.06935672461986542}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T10:15:31,612 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-09T10:15:31,612 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-09T10:15:31,613 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-09T10:15:31,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-09T10:15:31,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T10:15:31,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T10:15:31,615 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-09T10:15:31,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-09T10:15:31,622 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-09T10:15:31,622 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-09T10:15:31,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T10:15:31,646 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T10:15:31,656 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T10:15:31,811 DEBUG [de59e52b2fff:37695 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T10:15:31,818 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=de59e52b2fff,37621,1733739327100 2024-12-09T10:15:31,824 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as de59e52b2fff,37621,1733739327100, state=OPENING 2024-12-09T10:15:31,862 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T10:15:31,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:31,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:31,872 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T10:15:31,872 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T10:15:31,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=de59e52b2fff,37621,1733739327100}] 2024-12-09T10:15:32,061 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:32,064 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T10:15:32,070 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44266, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T10:15:32,099 INFO [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-09T10:15:32,100 INFO [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T10:15:32,101 INFO [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T10:15:32,117 INFO [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=de59e52b2fff%2C37621%2C1733739327100.meta, suffix=.meta, logDir=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/WALs/de59e52b2fff,37621,1733739327100, archiveDir=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/oldWALs, maxLogs=32 2024-12-09T10:15:32,142 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(617): When create output stream for /user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/WALs/de59e52b2fff,37621,1733739327100/de59e52b2fff%2C37621%2C1733739327100.meta.1733739332119.meta, exclude list is [], retry=0 2024-12-09T10:15:32,157 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34711,DS-99aab850-549c-4f09-862a-e7fc3422e5a2,DISK] 2024-12-09T10:15:32,171 INFO [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/WALs/de59e52b2fff,37621,1733739327100/de59e52b2fff%2C37621%2C1733739327100.meta.1733739332119.meta 2024-12-09T10:15:32,175 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44799:44799)] 2024-12-09T10:15:32,176 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:15:32,177 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T10:15:32,272 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T10:15:32,300 INFO [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T10:15:32,306 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T10:15:32,307 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:15:32,307 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-09T10:15:32,307 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-09T10:15:32,327 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T10:15:32,339 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T10:15:32,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:15:32,347 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:15:32,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T10:15:32,356 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T10:15:32,356 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:15:32,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:15:32,364 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T10:15:32,372 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T10:15:32,372 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:15:32,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:15:32,398 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740 2024-12-09T10:15:32,427 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740 2024-12-09T10:15:32,443 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T10:15:32,456 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-09T10:15:32,470 INFO [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62611493, jitterRate=-0.06701605021953583}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T10:15:32,473 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-09T10:15:32,492 INFO [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733739332055 2024-12-09T10:15:32,523 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=de59e52b2fff,37621,1733739327100 2024-12-09T10:15:32,526 DEBUG [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T10:15:32,527 INFO [RS_OPEN_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-09T10:15:32,531 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as de59e52b2fff,37621,1733739327100, state=OPEN 2024-12-09T10:15:32,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T10:15:32,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T10:15:32,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T10:15:32,602 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T10:15:32,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T10:15:32,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=de59e52b2fff,37621,1733739327100 in 724 msec 2024-12-09T10:15:32,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T10:15:32,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 989 msec 2024-12-09T10:15:32,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 2.2280 sec 2024-12-09T10:15:32,646 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733739332645, completionTime=-1 2024-12-09T10:15:32,646 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-09T10:15:32,646 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-09T10:15:32,717 DEBUG [hconnection-0x4d130334-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:15:32,733 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44270, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:15:32,758 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-09T10:15:32,759 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733739392758 2024-12-09T10:15:32,759 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733739452759 2024-12-09T10:15:32,759 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 112 msec 2024-12-09T10:15:32,839 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=de59e52b2fff,37695,1733739326017-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:32,840 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=de59e52b2fff,37695,1733739326017-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:32,840 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=de59e52b2fff,37695,1733739326017-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:32,843 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-de59e52b2fff:37695, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:32,847 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:32,856 DEBUG [master/de59e52b2fff:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-09T10:15:32,856 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-09T10:15:32,858 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T10:15:32,867 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-09T10:15:32,872 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T10:15:32,873 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:15:32,882 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T10:15:32,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741835_1011 (size=358) 2024-12-09T10:15:32,953 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => cb29027c56d4cda2e8d456ca713f729b, NAME => 'hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc 2024-12-09T10:15:33,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741836_1012 (size=42) 2024-12-09T10:15:33,021 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:15:33,021 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing cb29027c56d4cda2e8d456ca713f729b, disabling compactions & flushes 2024-12-09T10:15:33,021 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. 2024-12-09T10:15:33,021 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. 2024-12-09T10:15:33,021 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. after waiting 0 ms 2024-12-09T10:15:33,022 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. 2024-12-09T10:15:33,022 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. 2024-12-09T10:15:33,022 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for cb29027c56d4cda2e8d456ca713f729b: 2024-12-09T10:15:33,030 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T10:15:33,042 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733739333032"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733739333032"}]},"ts":"1733739333032"} 2024-12-09T10:15:33,106 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-09T10:15:33,112 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T10:15:33,121 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733739333112"}]},"ts":"1733739333112"} 2024-12-09T10:15:33,152 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-09T10:15:33,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=cb29027c56d4cda2e8d456ca713f729b, ASSIGN}] 2024-12-09T10:15:33,265 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=cb29027c56d4cda2e8d456ca713f729b, ASSIGN 2024-12-09T10:15:33,267 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=cb29027c56d4cda2e8d456ca713f729b, ASSIGN; state=OFFLINE, location=de59e52b2fff,37621,1733739327100; forceNewPlan=false, retain=false 2024-12-09T10:15:33,418 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=cb29027c56d4cda2e8d456ca713f729b, regionState=OPENING, regionLocation=de59e52b2fff,37621,1733739327100 2024-12-09T10:15:33,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure cb29027c56d4cda2e8d456ca713f729b, server=de59e52b2fff,37621,1733739327100}] 2024-12-09T10:15:33,578 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:33,584 INFO [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. 2024-12-09T10:15:33,585 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => cb29027c56d4cda2e8d456ca713f729b, NAME => 'hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:15:33,585 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace cb29027c56d4cda2e8d456ca713f729b 2024-12-09T10:15:33,585 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:15:33,586 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for cb29027c56d4cda2e8d456ca713f729b 2024-12-09T10:15:33,586 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for cb29027c56d4cda2e8d456ca713f729b 2024-12-09T10:15:33,589 INFO [StoreOpener-cb29027c56d4cda2e8d456ca713f729b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region cb29027c56d4cda2e8d456ca713f729b 2024-12-09T10:15:33,593 INFO [StoreOpener-cb29027c56d4cda2e8d456ca713f729b-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb29027c56d4cda2e8d456ca713f729b columnFamilyName info 2024-12-09T10:15:33,593 DEBUG [StoreOpener-cb29027c56d4cda2e8d456ca713f729b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:15:33,594 INFO [StoreOpener-cb29027c56d4cda2e8d456ca713f729b-1 {}] regionserver.HStore(327): Store=cb29027c56d4cda2e8d456ca713f729b/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:15:33,597 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/namespace/cb29027c56d4cda2e8d456ca713f729b 2024-12-09T10:15:33,598 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/namespace/cb29027c56d4cda2e8d456ca713f729b 2024-12-09T10:15:33,602 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for cb29027c56d4cda2e8d456ca713f729b 2024-12-09T10:15:33,608 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/namespace/cb29027c56d4cda2e8d456ca713f729b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T10:15:33,609 INFO [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened cb29027c56d4cda2e8d456ca713f729b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73701445, jitterRate=0.09823711216449738}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T10:15:33,611 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for cb29027c56d4cda2e8d456ca713f729b: 2024-12-09T10:15:33,616 INFO [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b., pid=6, masterSystemTime=1733739333578 2024-12-09T10:15:33,620 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. 2024-12-09T10:15:33,620 INFO [RS_OPEN_PRIORITY_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. 2024-12-09T10:15:33,626 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=cb29027c56d4cda2e8d456ca713f729b, regionState=OPEN, openSeqNum=2, regionLocation=de59e52b2fff,37621,1733739327100 2024-12-09T10:15:33,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T10:15:33,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure cb29027c56d4cda2e8d456ca713f729b, server=de59e52b2fff,37621,1733739327100 in 207 msec 2024-12-09T10:15:33,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T10:15:33,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=cb29027c56d4cda2e8d456ca713f729b, ASSIGN in 376 msec 2024-12-09T10:15:33,650 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T10:15:33,651 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733739333651"}]},"ts":"1733739333651"} 2024-12-09T10:15:33,661 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-09T10:15:33,681 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T10:15:33,688 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-09T10:15:33,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 822 msec 2024-12-09T10:15:33,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:33,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-09T10:15:33,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:33,748 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-09T10:15:33,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-09T10:15:33,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 61 msec 2024-12-09T10:15:33,815 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-09T10:15:33,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-09T10:15:33,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 47 msec 2024-12-09T10:15:33,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-09T10:15:33,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-09T10:15:33,907 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 6.663sec 2024-12-09T10:15:33,908 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T10:15:33,910 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T10:15:33,911 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T10:15:33,912 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T10:15:33,912 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T10:15:33,913 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=de59e52b2fff,37695,1733739326017-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T10:15:33,914 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=de59e52b2fff,37695,1733739326017-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T10:15:33,926 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7502c059 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6ea82e8f 2024-12-09T10:15:33,927 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-12-09T10:15:33,947 DEBUG [master/de59e52b2fff:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-09T10:15:33,949 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T10:15:33,949 INFO [master/de59e52b2fff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=de59e52b2fff,37695,1733739326017-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T10:15:33,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707adb81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:33,972 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T10:15:33,972 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T10:15:33,984 DEBUG [hconnection-0x3e1871-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:15:33,997 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44274, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:15:34,008 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=de59e52b2fff,37695,1733739326017 2024-12-09T10:15:34,010 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x57d89e0d to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d7b22aa 2024-12-09T10:15:34,031 DEBUG [ReadOnlyZKClient-127.0.0.1:64103@0x57d89e0d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8ebd37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:34,053 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=220, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=1289, ProcessCount=11, AvailableMemoryMB=4780 2024-12-09T10:15:34,093 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching master stub from registry 2024-12-09T10:15:34,102 DEBUG [ReadOnlyZKClient-127.0.0.1:64103@0x57d89e0d {}] client.AsyncConnectionImpl(310): The fetched master address is de59e52b2fff,37695,1733739326017 2024-12-09T10:15:34,105 DEBUG [ReadOnlyZKClient-127.0.0.1:64103@0x57d89e0d {}] client.ConnectionUtils(581): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b026c11 2024-12-09T10:15:34,107 DEBUG [ReadOnlyZKClient-127.0.0.1:64103@0x57d89e0d {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T10:15:34,128 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48524, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T10:15:34,130 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:15:34,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:15:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-09T10:15:34,181 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:34,190 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:34,218 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:34,223 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930004 connected 2024-12-09T10:15:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-09T10:15:34,342 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930004, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:34,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:34,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:15:34,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-09T10:15:34,560 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:34,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=10 2024-12-09T10:15:34,563 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:15:34,602 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=10}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:34,635 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-09T10:15:34,994 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-09T10:15:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=10 2024-12-09T10:15:35,001 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:35,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-09T10:15:35,005 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:15:35,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 609 msec 2024-12-09T10:15:35,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 873 msec 2024-12-09T10:15:35,035 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x68c65004 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3401485b 2024-12-09T10:15:35,064 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:35,064 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:35,065 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@22d222bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:35,068 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:35,111 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:35,113 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:35,115 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930006 connected 2024-12-09T10:15:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-09T10:15:35,290 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:15:35,293 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:15:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=11, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-12-09T10:15:35,323 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:35,325 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:35,373 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:35,379 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930007 connected 2024-12-09T10:15:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-12-09T10:15:35,532 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930007, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-12-09T10:15:35,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:35,782 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:35,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=12 2024-12-09T10:15:35,784 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:35,786 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:35,786 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:35,786 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:35,787 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930006, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:35,787 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930006, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-12-09T10:15:35,930 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930006, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:35,930 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=12}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x68c65004 to 127.0.0.1:64103 2024-12-09T10:15:35,930 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=12}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:35,955 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:35,956 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:35,957 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-09T10:15:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=12 2024-12-09T10:15:35,969 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:35,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=12, resume processing ppid=11 2024-12-09T10:15:35,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, ppid=11, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 343 msec 2024-12-09T10:15:35,980 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:15:35,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 687 msec 2024-12-09T10:15:36,010 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x139b8ec0 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1cf4bdf0 2024-12-09T10:15:36,068 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:36,069 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:36,070 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2bf3d98e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:36,072 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:36,102 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:36,103 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:36,107 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930009 connected 2024-12-09T10:15:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-12-09T10:15:36,450 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:36,455 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:36,458 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:36,459 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:36,464 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T10:15:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=13, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-12-09T10:15:36,490 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:36,492 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:36,527 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:36,531 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593000a connected 2024-12-09T10:15:36,576 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T10:15:36,576 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T10:15:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-12-09T10:15:36,589 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-12-09T10:15:36,589 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-12-09T10:15:36,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T10:15:36,592 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T10:15:36,593 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T10:15:36,593 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T10:15:36,648 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593000a, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:36,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-12-09T10:15:36,848 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:36,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=14 2024-12-09T10:15:36,850 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:36,852 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:36,852 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:36,852 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:36,852 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930009, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:36,853 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930009, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:36,935 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T10:15:36,937 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-09T10:15:36,979 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930009, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:36,979 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=14}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x139b8ec0 to 127.0.0.1:64103 2024-12-09T10:15:36,979 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=14}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:37,005 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:37,005 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:37,005 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-09T10:15:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=14 2024-12-09T10:15:37,023 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:37,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T10:15:37,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 330 msec 2024-12-09T10:15:37,030 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T10:15:37,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 566 msec 2024-12-09T10:15:37,052 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5807757f to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f88681b 2024-12-09T10:15:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-12-09T10:15:37,104 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:37,105 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:37,108 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:37,111 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:37,112 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:37,115 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-09T10:15:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-12-09T10:15:37,123 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:37,124 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:37,124 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@745158b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:37,126 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:37,155 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:37,160 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593000c connected 2024-12-09T10:15:37,160 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:37,161 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:37,162 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:37,219 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:37,223 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593000d connected 2024-12-09T10:15:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-12-09T10:15:37,371 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593000d, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:37,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-12-09T10:15:37,555 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:37,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=16 2024-12-09T10:15:37,557 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:37,559 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:37,559 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:37,559 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:37,559 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593000c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:37,559 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593000c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:37,674 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593000c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:37,674 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=16}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5807757f to 127.0.0.1:64103 2024-12-09T10:15:37,674 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=16}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:37,697 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:37,697 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:37,698 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-09T10:15:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=16 2024-12-09T10:15:37,711 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:37,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-12-09T10:15:37,718 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-09T10:15:37,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 310 msec 2024-12-09T10:15:37,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 604 msec 2024-12-09T10:15:37,742 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b9da59c to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@26ea60ee 2024-12-09T10:15:37,755 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:37,756 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:37,756 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@29b0bbb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:37,758 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:37,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-12-09T10:15:37,764 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:37,767 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:37,769 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:37,777 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:37,779 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:15:37,781 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=17, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:37,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593000f connected 2024-12-09T10:15:37,782 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-12-09T10:15:37,802 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:37,803 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:37,815 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:37,819 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930010 connected 2024-12-09T10:15:37,841 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:15:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-12-09T10:15:37,929 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930010, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:37,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:38,095 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:38,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=18 2024-12-09T10:15:38,096 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:38,098 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:38,098 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:38,098 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-12-09T10:15:38,098 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593000f, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:38,099 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593000f, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:38,212 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593000f, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:38,212 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=18}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2b9da59c to 127.0.0.1:64103 2024-12-09T10:15:38,213 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=18}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:38,233 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:38,234 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:38,234 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-09T10:15:38,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=18 2024-12-09T10:15:38,238 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:38,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=18, resume processing ppid=17 2024-12-09T10:15:38,243 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:15:38,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, ppid=17, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 296 msec 2024-12-09T10:15:38,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 465 msec 2024-12-09T10:15:38,278 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31e7eeb8 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@39b3b204 2024-12-09T10:15:38,299 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:38,300 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:38,300 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@76c3d14f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:38,302 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:38,314 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:38,317 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930012 connected 2024-12-09T10:15:38,319 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-12-09T10:15:38,410 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:38,411 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:38,412 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:38,414 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-09T10:15:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=19, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-12-09T10:15:38,435 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:38,436 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:38,445 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:38,446 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930013 connected 2024-12-09T10:15:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-12-09T10:15:38,557 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930013, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:38,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:38,720 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:38,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=20 2024-12-09T10:15:38,721 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:38,722 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:38,723 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:38,723 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:38,723 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930012, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:38,723 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930012, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-12-09T10:15:38,832 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930012, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:38,832 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=20}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x31e7eeb8 to 127.0.0.1:64103 2024-12-09T10:15:38,832 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=20}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:38,854 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:38,854 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:38,855 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-09T10:15:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=20 2024-12-09T10:15:38,859 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:38,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-12-09T10:15:38,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 292 msec 2024-12-09T10:15:38,864 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-09T10:15:38,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 450 msec 2024-12-09T10:15:38,891 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x38509a07 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@313b2acb 2024-12-09T10:15:38,933 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:38,933 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:38,933 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1b077f94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:38,935 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:38,945 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:38,946 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930015 connected 2024-12-09T10:15:38,947 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-12-09T10:15:39,050 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:39,051 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:39,052 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:15:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-12-09T10:15:39,076 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:39,077 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:39,091 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:39,099 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930016 connected 2024-12-09T10:15:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-12-09T10:15:39,211 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930016, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:39,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-12-09T10:15:39,395 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:39,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=22 2024-12-09T10:15:39,397 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:39,398 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:39,399 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:39,399 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:39,399 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930015, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:39,399 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930015, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:39,507 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=22}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x38509a07 to 127.0.0.1:64103 2024-12-09T10:15:39,508 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=22}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:39,508 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930015, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:39,545 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:39,545 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:39,546 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-09T10:15:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=22 2024-12-09T10:15:39,553 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:39,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-12-09T10:15:39,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 312 msec 2024-12-09T10:15:39,564 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:15:39,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 512 msec 2024-12-09T10:15:39,604 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x360623c4 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@ab48e89 2024-12-09T10:15:39,616 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:39,616 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:39,616 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@665143b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:39,617 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:39,629 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:39,630 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:39,635 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930018 connected 2024-12-09T10:15:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-12-09T10:15:39,680 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:39,681 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:39,685 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:15:39,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:15:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=23, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T10:15:39,692 DEBUG [PEWorker-2 {}] procedure.MasterProcedureScheduler(583): PEER '1', shared lock count=0, exclusively locked by procId=23 2024-12-09T10:15:39,692 DEBUG [PEWorker-2 {}] procedure2.ProcedureExecutor(1470): LOCK_EVENT_WAIT pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-09T10:15:39,695 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:15:39,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:15:39,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:15:39,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:15:39,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=23, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T10:15:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-09T10:15:39,858 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:39,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=25 2024-12-09T10:15:39,860 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:15:39,860 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:15:39,860 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:15:39,860 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:39,860 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930018, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:39,860 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930018, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:39,974 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930018, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:39,974 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=25}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x360623c4 to 127.0.0.1:64103 2024-12-09T10:15:39,974 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=25}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:39,975 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:15:39,988 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=25}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:15:39,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:15:39,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:15:39,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:15:40,000 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=25}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:15:40,000 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=25 2024-12-09T10:15:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=25 2024-12-09T10:15:40,004 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:40,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=23 2024-12-09T10:15:40,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=23, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 299 msec 2024-12-09T10:15:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T10:15:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-09T10:15:40,021 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:15:40,021 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:15:40,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 336 msec 2024-12-09T10:15:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T10:15:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-09T10:15:40,320 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:15:40,320 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:15:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=26, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-12-09T10:15:40,323 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:40,331 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=26, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:40,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=26, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=12 msec 2024-12-09T10:15:40,334 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-12-09T10:15:40,431 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:15:40,432 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:40,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:40,441 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x100be8885930019 connected 2024-12-09T10:15:40,526 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=221 (was 220) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=452 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1322 (was 1289) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3756 (was 4780) 2024-12-09T10:15:40,540 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=221, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=1322, ProcessCount=11, AvailableMemoryMB=3754 2024-12-09T10:15:40,543 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:15:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:15:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-12-09T10:15:40,570 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:40,572 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:40,584 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:40,587 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593001a connected 2024-12-09T10:15:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-12-09T10:15:40,713 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593001a, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T10:15:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-12-09T10:15:40,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:41,073 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:41,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=28 2024-12-09T10:15:41,074 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:15:41,094 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=28}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:41,112 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-12-09T10:15:41,182 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-09T10:15:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=28 2024-12-09T10:15:41,186 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:41,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-12-09T10:15:41,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 268 msec 2024-12-09T10:15:41,191 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:15:41,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 647 msec 2024-12-09T10:15:41,216 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x237b2dd9 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3098f115 2024-12-09T10:15:41,238 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:41,238 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:41,238 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@46c5b6c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:41,240 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:41,254 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:41,255 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593001c connected 2024-12-09T10:15:41,256 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-12-09T10:15:41,689 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:15:41,690 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:15:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=29, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-12-09T10:15:41,706 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:41,707 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:41,715 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:41,716 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593001d connected 2024-12-09T10:15:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-12-09T10:15:41,824 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593001d, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T10:15:41,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:42,002 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:42,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=30 2024-12-09T10:15:42,003 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:42,005 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:42,005 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:42,005 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:42,005 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593001c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:42,005 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593001c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-12-09T10:15:42,120 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593001c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:42,121 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=30}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x237b2dd9 to 127.0.0.1:64103 2024-12-09T10:15:42,121 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=30}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:42,141 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:42,141 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:42,142 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-12-09T10:15:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=30 2024-12-09T10:15:42,148 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:42,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=30, resume processing ppid=29 2024-12-09T10:15:42,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, ppid=29, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 300 msec 2024-12-09T10:15:42,153 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:15:42,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 463 msec 2024-12-09T10:15:42,186 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0f21ace1 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5c2894c2 2024-12-09T10:15:42,216 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:42,217 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:42,217 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4f3f18b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:42,219 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:42,246 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:42,251 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593001f connected 2024-12-09T10:15:42,251 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-12-09T10:15:42,320 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:42,321 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:42,323 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T10:15:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=31, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-12-09T10:15:42,348 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:42,349 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:42,358 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:42,362 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930020 connected 2024-12-09T10:15:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-12-09T10:15:42,482 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930020, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:42,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:42,649 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-12-09T10:15:42,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=32 2024-12-09T10:15:42,650 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:42,651 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:42,651 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:42,651 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:42,651 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593001f, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:42,651 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593001f, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:42,762 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593001f, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:42,762 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=32}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0f21ace1 to 127.0.0.1:64103 2024-12-09T10:15:42,762 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=32}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:42,781 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:42,781 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:42,782 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32 2024-12-09T10:15:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=32 2024-12-09T10:15:42,786 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:42,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-12-09T10:15:42,792 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T10:15:42,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 291 msec 2024-12-09T10:15:42,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 469 msec 2024-12-09T10:15:42,824 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d32b564 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3fe98780 2024-12-09T10:15:42,851 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:42,852 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:42,852 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5988905e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:42,857 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:42,870 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:42,871 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:42,877 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930022 connected 2024-12-09T10:15:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-12-09T10:15:42,960 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:42,960 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:42,962 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-12-09T10:15:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-12-09T10:15:42,984 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:42,985 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:42,999 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:43,001 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930023 connected 2024-12-09T10:15:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-12-09T10:15:43,115 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930023, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:43,116 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:43,118 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=33, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:43,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=33, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=156 msec 2024-12-09T10:15:43,120 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-12-09T10:15:43,281 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-12-09T10:15:43,282 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:43,283 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-09T10:15:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=34, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-09T10:15:43,310 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:43,311 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:43,333 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:43,335 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930024 connected 2024-12-09T10:15:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-09T10:15:43,453 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930024, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:43,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-09T10:15:43,622 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:43,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=35 2024-12-09T10:15:43,623 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:43,624 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:43,624 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:43,625 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:43,625 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930022, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:43,625 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930022, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:43,737 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930022, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:43,737 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=35}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7d32b564 to 127.0.0.1:64103 2024-12-09T10:15:43,737 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=35}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:43,759 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:43,759 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:43,760 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-09T10:15:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=35 2024-12-09T10:15:43,779 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:43,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-12-09T10:15:43,804 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-09T10:15:43,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 313 msec 2024-12-09T10:15:43,806 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1897a5e9 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@71ccc3f6 2024-12-09T10:15:43,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 521 msec 2024-12-09T10:15:43,843 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:43,844 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:43,844 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@68376d1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:43,846 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T10:15:43,865 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:43,869 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930026 connected 2024-12-09T10:15:43,872 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-09T10:15:43,920 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:43,926 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:43,938 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-09T10:15:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-09T10:15:43,964 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:43,965 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:43,984 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:43,985 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930027 connected 2024-12-09T10:15:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-09T10:15:44,095 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930027, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:44,095 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:44,098 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=36, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:44,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=36, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=160 msec 2024-12-09T10:15:44,100 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-09T10:15:44,261 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-12-09T10:15:44,261 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:15:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=37, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-12-09T10:15:44,267 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:15:44,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:15:44,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:15:44,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:15:44,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-12-09T10:15:44,433 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:44,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=38 2024-12-09T10:15:44,434 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:15:44,434 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:15:44,435 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:15:44,435 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:44,435 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930026, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:44,435 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930026, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:44,545 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930026, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:44,547 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=38}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1897a5e9 to 127.0.0.1:64103 2024-12-09T10:15:44,547 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=38}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:44,547 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:15:44,561 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=38}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:15:44,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:15:44,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:15:44,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:15:44,581 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=38}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:15:44,581 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=38 2024-12-09T10:15:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-12-09T10:15:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=38 2024-12-09T10:15:44,587 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:44,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-12-09T10:15:44,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 307 msec 2024-12-09T10:15:44,626 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:15:44,626 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:15:44,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 365 msec 2024-12-09T10:15:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-12-09T10:15:44,893 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:15:44,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:15:44,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:44,902 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:44,905 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=39, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:44,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-12-09T10:15:44,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=39, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=12 msec 2024-12-09T10:15:44,908 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-12-09T10:15:45,020 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:15:45,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:15:45,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=40, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40 2024-12-09T10:15:45,033 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:45,035 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=40, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:45,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=40, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=14 msec 2024-12-09T10:15:45,040 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40 2024-12-09T10:15:45,142 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:15:45,175 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=218 (was 221), OpenFileDescriptor=446 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1288 (was 1322), ProcessCount=11 (was 11), AvailableMemoryMB=3417 (was 3754) 2024-12-09T10:15:45,195 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=218, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=1288, ProcessCount=11, AvailableMemoryMB=3416 2024-12-09T10:15:45,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:15:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=41, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:15:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-09T10:15:45,275 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:45,286 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:45,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-09T10:15:45,341 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:45,344 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930028 connected 2024-12-09T10:15:45,459 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930028, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:45,485 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:15:45,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-09T10:15:45,661 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:45,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=42 2024-12-09T10:15:45,662 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:15:45,724 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=42}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:45,774 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-09T10:15:45,860 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=42 2024-12-09T10:15:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=42 2024-12-09T10:15:45,872 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:45,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=42, resume processing ppid=41 2024-12-09T10:15:45,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, ppid=41, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 372 msec 2024-12-09T10:15:45,883 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:15:45,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 684 msec 2024-12-09T10:15:45,938 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ce8419d to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4e1da0ea 2024-12-09T10:15:45,976 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:45,977 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:45,977 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5fe02d74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:45,978 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:45,999 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:46,001 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593002a connected 2024-12-09T10:15:46,005 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-09T10:15:46,361 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:15:46,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:15:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=43, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:15:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-12-09T10:15:46,397 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:46,398 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-12-09T10:15:46,487 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:46,487 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593002b connected 2024-12-09T10:15:46,604 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:46,605 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593002b, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:46,611 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=43, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:46,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=43, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=247 msec 2024-12-09T10:15:46,613 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-12-09T10:15:46,692 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-12-09T10:15:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:15:46,710 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:15:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=44, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=44 2024-12-09T10:15:46,723 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:46,726 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=44, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:46,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=44, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=16 msec 2024-12-09T10:15:46,733 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:46,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=44 2024-12-09T10:15:46,831 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:15:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:15:46,839 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=127.0.0.1:64103:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:15:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:15:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-12-09T10:15:46,870 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:46,873 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:46,896 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:46,898 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593002c connected 2024-12-09T10:15:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-12-09T10:15:47,010 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-12-09T10:15:47,016 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593002c, quorum=127.0.0.1:64103, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:47,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-12-09T10:15:47,216 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:47,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=46 2024-12-09T10:15:47,217 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=ADD_PEER 2024-12-09T10:15:47,236 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=46}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:47,256 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSource(229): queueId=2, ReplicationSource: 2, currentBandwidth=0 2024-12-09T10:15:47,282 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46 2024-12-09T10:15:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=46 2024-12-09T10:15:47,286 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:47,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-12-09T10:15:47,289 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 2, config clusterKey=127.0.0.1:64103:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:15:47,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 231 msec 2024-12-09T10:15:47,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 450 msec 2024-12-09T10:15:47,321 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x08eb5ab9 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@123a7188 2024-12-09T10:15:47,349 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:47,349 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:47,349 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@537e6062, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:47,351 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 2 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:47,365 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 20x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:47,366 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 2-0x100be888593002e connected 2024-12-09T10:15:47,368 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] regionserver.ReplicationSource(569): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-12-09T10:15:47,480 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-12-09T10:15:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:15:47,482 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:15:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=47, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-12-09T10:15:47,489 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:15:47,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:15:47,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:15:47,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:15:47,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-12-09T10:15:47,656 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:47,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=48 2024-12-09T10:15:47,658 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:15:47,658 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:15:47,658 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:15:47,658 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:47,658 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593002a, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:47,658 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593002a, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:47,770 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593002a, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:47,771 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=48}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6ce8419d to 127.0.0.1:64103 2024-12-09T10:15:47,771 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=48}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:47,771 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:15:47,782 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=48}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:15:47,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:15:47,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:15:47,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:15:47,795 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=48}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:15:47,796 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-12-09T10:15:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=48 2024-12-09T10:15:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-12-09T10:15:47,802 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:47,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=48, resume processing ppid=47 2024-12-09T10:15:47,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, ppid=47, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 302 msec 2024-12-09T10:15:47,874 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:15:47,874 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:15:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T10:15:47,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 392 msec 2024-12-09T10:15:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-12-09T10:15:48,110 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:15:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:15:48,112 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:15:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=49, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-09T10:15:48,130 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-12-09T10:15:48,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-12-09T10:15:48,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-09T10:15:48,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-09T10:15:48,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-09T10:15:48,320 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:48,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=50 2024-12-09T10:15:48,321 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-12-09T10:15:48,321 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 2: 0 2024-12-09T10:15:48,322 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSource(657): peerId=2, Closing source 2 because: Replication stream was removed by a user 2024-12-09T10:15:48,322 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] regionserver.ReplicationSource(508): peerId=2, Interrupted while sleeping between retries 2024-12-09T10:15:48,322 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] zookeeper.ZKUtil(165): connection to cluster: 2-0x100be888593002e, quorum=127.0.0.1:64103, baseZNode=/hbase-test2 Unable to set watcher on znode (/hbase-test2/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:48,322 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] zookeeper.ZKWatcher(734): connection to cluster: 2-0x100be888593002e, quorum=127.0.0.1:64103, baseZNode=/hbase-test2 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-09T10:15:48,450 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 2-0x100be888593002e, quorum=127.0.0.1:64103, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:48,450 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=50}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x08eb5ab9 to 127.0.0.1:64103 2024-12-09T10:15:48,450 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=50}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:48,451 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSourceManager(430): Done with the queue 2 2024-12-09T10:15:48,478 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=50}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/2/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:15:48,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/2/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:15:48,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/2 2024-12-09T10:15:48,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/2 2024-12-09T10:15:48,522 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=50}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-12-09T10:15:48,522 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=50 2024-12-09T10:15:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=50 2024-12-09T10:15:48,535 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:48,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-12-09T10:15:48,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 374 msec 2024-12-09T10:15:48,583 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-12-09T10:15:48,583 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 2 2024-12-09T10:15:48,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 471 msec 2024-12-09T10:15:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-09T10:15:48,750 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-12-09T10:15:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:15:48,757 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:15:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-12-09T10:15:48,771 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:48,778 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=51, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:48,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=51, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=22 msec 2024-12-09T10:15:48,785 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-12-09T10:15:48,881 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:15:48,884 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:15:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=52, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=52 2024-12-09T10:15:48,904 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:48,906 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=52, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:48,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=52, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=22 msec 2024-12-09T10:15:48,913 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=52 2024-12-09T10:15:49,010 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:15:49,046 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=219 (was 218) - Thread LEAK? -, OpenFileDescriptor=444 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1225 (was 1288), ProcessCount=11 (was 11), AvailableMemoryMB=3920 (was 3416) - AvailableMemoryMB LEAK? - 2024-12-09T10:15:49,067 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=1225, ProcessCount=11, AvailableMemoryMB=3918 2024-12-09T10:15:49,078 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:15:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=53, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:15:49,083 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] ... 12 more 2024-12-09T10:15:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-12-09T10:15:49,087 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=53, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:49,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=53, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=10 msec 2024-12-09T10:15:49,090 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-12-09T10:15:49,201 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-12-09T10:15:49,207 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:15:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-12-09T10:15:49,219 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:49,222 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=54, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:49,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=54, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=15 msec 2024-12-09T10:15:49,231 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-12-09T10:15:49,321 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:15:49,322 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:15:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=55, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-12-09T10:15:49,336 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:49,340 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=55, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:49,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=55, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=18 msec 2024-12-09T10:15:49,345 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-12-09T10:15:49,440 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:15:49,480 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=219 (was 219), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1225 (was 1225), ProcessCount=11 (was 11), AvailableMemoryMB=3899 (was 3918) 2024-12-09T10:15:49,504 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=219, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=1225, ProcessCount=11, AvailableMemoryMB=3898 2024-12-09T10:15:49,505 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:15:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=56, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:15:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=56 2024-12-09T10:15:49,526 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-12-09T10:15:49,537 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=56, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:49,548 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=56, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=33 msec 2024-12-09T10:15:49,548 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=56 2024-12-09T10:15:49,644 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-12-09T10:15:49,645 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:15:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-12-09T10:15:49,667 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:49,683 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=57, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:49,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=57, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=39 msec 2024-12-09T10:15:49,693 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-12-09T10:15:49,764 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:15:49,765 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:15:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=58, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:49,779 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=58 2024-12-09T10:15:49,783 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=58, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:49,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=58, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=19 msec 2024-12-09T10:15:49,791 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=58 2024-12-09T10:15:49,890 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:15:49,915 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=219 (was 219), OpenFileDescriptor=444 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1225 (was 1225), ProcessCount=11 (was 11), AvailableMemoryMB=3887 (was 3898) 2024-12-09T10:15:49,929 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=1225, ProcessCount=11, AvailableMemoryMB=3886 2024-12-09T10:15:49,930 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:15:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=59, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:15:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-09T10:15:49,963 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:49,967 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:50,036 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-09T10:15:50,043 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593002f connected 2024-12-09T10:15:50,173 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593002f, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:50,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-09T10:15:50,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:50,459 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:50,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=60 2024-12-09T10:15:50,460 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:15:50,535 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=60}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:50,558 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:50,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-09T10:15:50,800 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-12-09T10:15:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=60 2024-12-09T10:15:50,805 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:50,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=60, resume processing ppid=59 2024-12-09T10:15:50,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, ppid=59, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 504 msec 2024-12-09T10:15:50,810 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:15:50,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 880 msec 2024-12-09T10:15:50,852 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x071d27f0 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@57eee1b4 2024-12-09T10:15:50,899 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:50,899 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:50,900 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7e6523a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:50,901 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:50,940 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:50,940 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:50,940 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930031 connected 2024-12-09T10:15:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-09T10:15:51,070 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:15:51,070 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:15:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=61, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-09T10:15:51,096 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:51,097 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:51,113 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:51,118 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930032 connected 2024-12-09T10:15:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-09T10:15:51,249 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930032, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-09T10:15:51,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:51,604 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:51,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=62 2024-12-09T10:15:51,605 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:51,606 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:51,606 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:51,606 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:51,606 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930031, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:51,607 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930031, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-09T10:15:51,715 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930031, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:51,715 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=62}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x071d27f0 to 127.0.0.1:64103 2024-12-09T10:15:51,715 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=62}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:51,739 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:51,740 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:51,740 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-12-09T10:15:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=62 2024-12-09T10:15:51,752 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:51,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-12-09T10:15:51,757 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:15:51,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 302 msec 2024-12-09T10:15:51,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 687 msec 2024-12-09T10:15:51,799 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x02ddd560 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7f5480f6 2024-12-09T10:15:51,851 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:51,852 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:51,852 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@676c9a57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:51,853 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:51,944 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:51,945 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:51,945 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930034 connected 2024-12-09T10:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-09T10:15:52,209 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:52,209 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:52,210 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-09T10:15:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-09T10:15:52,233 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:52,233 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:52,291 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:52,294 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930035 connected 2024-12-09T10:15:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-09T10:15:52,424 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930035, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:52,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-09T10:15:52,592 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:52,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=64 2024-12-09T10:15:52,596 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:52,599 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:52,599 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:52,599 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:52,600 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930034, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:52,600 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930034, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:52,720 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930034, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:52,720 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=64}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x02ddd560 to 127.0.0.1:64103 2024-12-09T10:15:52,720 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=64}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:52,755 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:52,756 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:52,759 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=64 2024-12-09T10:15:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=64 2024-12-09T10:15:52,771 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:52,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-12-09T10:15:52,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 333 msec 2024-12-09T10:15:52,784 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-09T10:15:52,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 574 msec 2024-12-09T10:15:52,835 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x25851c31 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4b2ebf36 2024-12-09T10:15:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-09T10:15:52,845 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:52,846 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:52,848 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:52,849 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T10:15:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=65, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-09T10:15:52,872 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:52,873 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:52,873 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1ea4757f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:52,875 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:52,903 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:52,904 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930037 connected 2024-12-09T10:15:52,908 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:52,910 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:52,919 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-09T10:15:52,972 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:52,973 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930038 connected 2024-12-09T10:15:53,119 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930038, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:53,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-09T10:15:53,280 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:15:53,315 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:53,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=66 2024-12-09T10:15:53,317 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:53,339 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:53,339 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:53,343 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:53,343 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930037, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:53,343 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930037, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:53,479 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930037, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:53,479 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=66}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x25851c31 to 127.0.0.1:64103 2024-12-09T10:15:53,479 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=66}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:53,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-09T10:15:53,510 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:53,510 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:53,511 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-12-09T10:15:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=66 2024-12-09T10:15:53,524 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:53,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=66, resume processing ppid=65 2024-12-09T10:15:53,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, ppid=65, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 369 msec 2024-12-09T10:15:53,529 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T10:15:53,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 679 msec 2024-12-09T10:15:53,556 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6575e1b8 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@37964596 2024-12-09T10:15:53,585 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:53,586 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:53,587 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@d8e4c11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:53,595 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:53,625 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:53,627 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:53,631 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593003a connected 2024-12-09T10:15:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-09T10:15:53,999 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:54,000 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:54,003 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:15:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=67, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-09T10:15:54,011 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:15:54,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:15:54,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:15:54,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:15:54,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-09T10:15:54,193 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:54,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=68 2024-12-09T10:15:54,195 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:15:54,195 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:15:54,195 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:15:54,195 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:54,196 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593003a, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:54,196 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593003a, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:54,307 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593003a, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:54,307 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=68}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6575e1b8 to 127.0.0.1:64103 2024-12-09T10:15:54,307 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=68}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:54,307 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:15:54,315 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=68}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:15:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-09T10:15:54,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:15:54,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:15:54,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:15:54,332 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=68}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:15:54,332 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=68 2024-12-09T10:15:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=68 2024-12-09T10:15:54,336 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:54,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-12-09T10:15:54,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 300 msec 2024-12-09T10:15:54,417 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:15:54,417 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:15:54,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 414 msec 2024-12-09T10:15:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-09T10:15:54,630 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:15:54,631 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:15:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-12-09T10:15:54,638 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:54,642 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=69, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:54,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=69, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=11 msec 2024-12-09T10:15:54,647 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-12-09T10:15:54,752 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:15:54,753 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:15:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=70, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:15:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=70 2024-12-09T10:15:54,764 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:15:54,771 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=70, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:15:54,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=70, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=19 msec 2024-12-09T10:15:54,774 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:15:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=70 2024-12-09T10:15:54,871 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:15:54,908 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=217 (was 219), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1247 (was 1225) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3274 (was 3886) 2024-12-09T10:15:54,918 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T10:15:54,918 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=217, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=1247, ProcessCount=11, AvailableMemoryMB=3273 2024-12-09T10:15:54,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:15:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=71, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:15:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-09T10:15:54,937 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:54,937 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:54,949 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:54,950 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593003b connected 2024-12-09T10:15:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-09T10:15:55,056 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593003b, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:55,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:55,236 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:55,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=72 2024-12-09T10:15:55,237 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:15:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-09T10:15:55,260 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=72}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:55,282 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:55,331 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72 2024-12-09T10:15:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=72 2024-12-09T10:15:55,337 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:55,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=72, resume processing ppid=71 2024-12-09T10:15:55,340 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:15:55,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, ppid=71, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 255 msec 2024-12-09T10:15:55,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 421 msec 2024-12-09T10:15:55,373 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5c06e288 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@249067c9 2024-12-09T10:15:55,388 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:55,389 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:55,389 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@17bf9383, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:55,391 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:55,403 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:55,404 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593003d connected 2024-12-09T10:15:55,408 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-09T10:15:55,556 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:15:55,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:15:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=73, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-12-09T10:15:55,594 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:55,594 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:55,618 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:55,623 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593003e connected 2024-12-09T10:15:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-12-09T10:15:55,753 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593003e, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:55,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-12-09T10:15:55,931 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:55,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=74 2024-12-09T10:15:55,934 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:55,942 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:55,943 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:55,943 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:55,943 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593003d, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:55,943 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593003d, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:56,064 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593003d, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:56,064 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=74}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5c06e288 to 127.0.0.1:64103 2024-12-09T10:15:56,064 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=74}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:56,085 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:56,086 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:56,086 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74 2024-12-09T10:15:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=74 2024-12-09T10:15:56,090 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:56,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-12-09T10:15:56,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 316 msec 2024-12-09T10:15:56,096 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:15:56,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 539 msec 2024-12-09T10:15:56,143 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58073a22 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7df12f78 2024-12-09T10:15:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-12-09T10:15:56,201 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:56,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:56,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T10:15:56,203 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:56,204 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:56,204 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7f8cd59a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:56,205 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-09T10:15:56,230 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:56,232 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:56,234 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930040 connected 2024-12-09T10:15:56,248 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:56,249 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:56,287 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:56,293 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930041 connected 2024-12-09T10:15:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-09T10:15:56,409 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930041, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:56,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-09T10:15:56,615 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:56,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=76 2024-12-09T10:15:56,618 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:56,620 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:56,620 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:56,621 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:56,622 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930040, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:56,623 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930040, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-09T10:15:56,923 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=76}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x58073a22 to 127.0.0.1:64103 2024-12-09T10:15:56,923 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=76}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:56,925 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930040, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:56,947 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:56,948 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:56,951 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76 2024-12-09T10:15:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=76 2024-12-09T10:15:56,961 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:56,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-12-09T10:15:56,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 514 msec 2024-12-09T10:15:56,971 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T10:15:56,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 768 msec 2024-12-09T10:15:57,031 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6077f833 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a69e70b 2024-12-09T10:15:57,075 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:57,076 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:57,076 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@576b1aef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:57,084 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:57,098 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:57,098 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930043 connected 2024-12-09T10:15:57,107 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-09T10:15:57,364 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:57,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:57,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:57,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T10:15:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=77, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-09T10:15:57,416 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:57,417 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:57,478 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:57,492 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930044 connected 2024-12-09T10:15:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-09T10:15:57,607 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930044, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:57,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-09T10:15:57,790 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:57,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=78 2024-12-09T10:15:57,791 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:57,799 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:57,799 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:57,799 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:57,800 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930043, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:57,800 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930043, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:57,915 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=78}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6077f833 to 127.0.0.1:64103 2024-12-09T10:15:57,915 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=78}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:57,918 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930043, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:57,936 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:57,936 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:57,937 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78 2024-12-09T10:15:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=78 2024-12-09T10:15:57,946 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:57,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T10:15:57,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=78, resume processing ppid=77 2024-12-09T10:15:57,959 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T10:15:57,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, ppid=77, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 315 msec 2024-12-09T10:15:57,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 590 msec 2024-12-09T10:15:57,989 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5e816d0f to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7acc7221 2024-12-09T10:15:58,016 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:58,017 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:58,017 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@73dbf4a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:58,018 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:58,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-09T10:15:58,027 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:58,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:58,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:58,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T10:15:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=79, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:58,034 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-09T10:15:58,038 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930046 connected 2024-12-09T10:15:58,050 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:58,061 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:58,062 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:58,073 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-09T10:15:58,188 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:58,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-09T10:15:58,373 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:58,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=80 2024-12-09T10:15:58,374 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:58,376 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:58,376 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:58,376 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:58,376 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930046, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:58,377 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930046, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:58,487 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930046, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:58,487 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=80}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5e816d0f to 127.0.0.1:64103 2024-12-09T10:15:58,487 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=80}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:58,507 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:58,508 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:58,509 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80 2024-12-09T10:15:58,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=80 2024-12-09T10:15:58,514 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:58,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-12-09T10:15:58,519 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T10:15:58,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 300 msec 2024-12-09T10:15:58,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 488 msec 2024-12-09T10:15:58,558 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a3d0724 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1308ac86 2024-12-09T10:15:58,582 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:58,583 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:58,583 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4e1faa8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:58,584 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:58,610 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:58,611 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:58,616 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930049 connected 2024-12-09T10:15:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-09T10:15:58,672 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:58,673 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:58,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:58,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T10:15:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-09T10:15:58,714 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:58,716 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:58,735 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-09T10:15:58,844 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:58,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-09T10:15:59,023 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:59,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=82 2024-12-09T10:15:59,029 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:59,041 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:59,041 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:59,047 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:59,047 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930049, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:59,048 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930049, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:59,175 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930049, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:59,175 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=82}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a3d0724 to 127.0.0.1:64103 2024-12-09T10:15:59,175 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=82}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:59,220 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:59,221 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:59,227 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-12-09T10:15:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=82 2024-12-09T10:15:59,233 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:59,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-12-09T10:15:59,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 365 msec 2024-12-09T10:15:59,237 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T10:15:59,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 559 msec 2024-12-09T10:15:59,282 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x689de55a to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@156e03c3 2024-12-09T10:15:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-09T10:15:59,320 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:59,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:59,320 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:59,321 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:59,321 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@37fd3345, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:59,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:59,322 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:59,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T10:15:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=83, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-09T10:15:59,356 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:59,357 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593004c connected 2024-12-09T10:15:59,357 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:59,364 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:15:59,365 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:59,399 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:59,400 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593004d connected 2024-12-09T10:15:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-09T10:15:59,531 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593004d, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:59,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:15:59,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-09T10:15:59,703 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:15:59,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=84 2024-12-09T10:15:59,704 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:15:59,705 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:15:59,705 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:15:59,706 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:15:59,706 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593004c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:59,706 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593004c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:15:59,810 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:15:59,815 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593004c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:15:59,815 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=84}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x689de55a to 127.0.0.1:64103 2024-12-09T10:15:59,815 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=84}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:15:59,842 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:15:59,842 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:15:59,844 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-12-09T10:15:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=84 2024-12-09T10:15:59,848 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:15:59,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=84, resume processing ppid=83 2024-12-09T10:15:59,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, ppid=83, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 305 msec 2024-12-09T10:15:59,851 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T10:15:59,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 526 msec 2024-12-09T10:15:59,910 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x62b5ddd9 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d4b153e 2024-12-09T10:15:59,943 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:15:59,944 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:15:59,944 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2f763105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:15:59,946 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:15:59,957 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:15:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-09T10:15:59,962 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:15:59,963 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593004f connected 2024-12-09T10:15:59,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:15:59,968 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:15:59,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T10:15:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=85, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:15:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-09T10:16:00,004 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:00,005 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:00,035 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:00,048 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930050 connected 2024-12-09T10:16:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-09T10:16:00,160 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930050, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:00,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-09T10:16:00,363 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:00,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=86 2024-12-09T10:16:00,371 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:00,387 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-12-09T10:16:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=86 2024-12-09T10:16:00,393 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:00,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-12-09T10:16:00,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec 2024-12-09T10:16:00,400 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T10:16:00,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 432 msec 2024-12-09T10:16:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-09T10:16:00,600 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:00,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:00,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:00,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-09T10:16:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-09T10:16:00,629 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:00,630 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:00,653 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:00,659 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930051 connected 2024-12-09T10:16:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-09T10:16:00,792 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930051, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:00,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-09T10:16:00,967 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:00,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=88 2024-12-09T10:16:00,967 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:00,971 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-12-09T10:16:00,984 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:00,984 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:00,985 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:00,985 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593004f, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:00,985 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593004f, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:01,104 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593004f, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:01,104 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=88}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x62b5ddd9 to 127.0.0.1:64103 2024-12-09T10:16:01,104 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=88}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:01,122 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:01,122 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:01,122 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-12-09T10:16:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=88 2024-12-09T10:16:01,125 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:01,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-12-09T10:16:01,128 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-09T10:16:01,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 312 msec 2024-12-09T10:16:01,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 525 msec 2024-12-09T10:16:01,161 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x67af9186 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@56849a91 2024-12-09T10:16:01,178 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:01,179 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:01,179 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@613794d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:01,180 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:01,189 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:01,190 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:01,190 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930053 connected 2024-12-09T10:16:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-09T10:16:01,230 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:01,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:01,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-09T10:16:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=89, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-09T10:16:01,251 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:01,252 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:01,270 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:01,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930054 connected 2024-12-09T10:16:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-09T10:16:01,384 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930054, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:01,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-09T10:16:01,573 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:01,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=90 2024-12-09T10:16:01,580 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:01,581 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:01,581 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:01,581 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:01,581 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930053, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:01,581 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930053, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:01,695 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930053, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:01,696 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=90}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x67af9186 to 127.0.0.1:64103 2024-12-09T10:16:01,696 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=90}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:01,716 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:01,716 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:01,717 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-09T10:16:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=90 2024-12-09T10:16:01,721 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:01,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=90, resume processing ppid=89 2024-12-09T10:16:01,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, ppid=89, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 301 msec 2024-12-09T10:16:01,725 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-09T10:16:01,727 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 493 msec 2024-12-09T10:16:01,750 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x25c59bc3 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55260972 2024-12-09T10:16:01,862 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:01,862 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:01,862 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3f6a3ed7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:01,864 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-09T10:16:01,869 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:01,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:01,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=91, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-09T10:16:01,903 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:01,903 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:01,903 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:01,903 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930056 connected 2024-12-09T10:16:01,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:01,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:01,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:01,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-09T10:16:02,077 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:02,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=92 2024-12-09T10:16:02,078 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:02,078 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:02,078 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:02,078 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:02,078 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930056, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:02,079 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930056, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-09T10:16:02,189 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930056, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:02,190 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=92}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x25c59bc3 to 127.0.0.1:64103 2024-12-09T10:16:02,190 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=92}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:02,190 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:02,198 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=92}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:02,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:02,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:02,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:02,206 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=92}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:02,206 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-09T10:16:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=92 2024-12-09T10:16:02,209 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:02,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-12-09T10:16:02,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 285 msec 2024-12-09T10:16:02,223 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:02,223 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:02,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 353 msec 2024-12-09T10:16:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-09T10:16:02,499 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:02,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:02,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=93, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:02,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-12-09T10:16:02,501 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:02,502 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=93, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:02,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=93, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T10:16:02,503 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-12-09T10:16:02,610 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:02,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=94, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-12-09T10:16:02,612 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:02,613 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=94, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:02,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=94, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T10:16:02,615 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:02,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-12-09T10:16:02,720 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:02,731 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=221 (was 217) - Thread LEAK? -, OpenFileDescriptor=448 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1279 (was 1247) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2674 (was 3273) 2024-12-09T10:16:02,739 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=221, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=1279, ProcessCount=11, AvailableMemoryMB=2674 2024-12-09T10:16:02,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=95, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-12-09T10:16:02,759 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:02,760 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:02,794 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:02,795 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930057 connected 2024-12-09T10:16:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-12-09T10:16:02,903 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930057, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:02,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-12-09T10:16:03,076 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:03,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=96 2024-12-09T10:16:03,077 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:03,101 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=96}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:03,124 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:03,164 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-09T10:16:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=96 2024-12-09T10:16:03,168 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:03,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=96, resume processing ppid=95 2024-12-09T10:16:03,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=96, ppid=95, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 244 msec 2024-12-09T10:16:03,173 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:03,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=95, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 434 msec 2024-12-09T10:16:03,208 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x43da105b to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6640a8b5 2024-12-09T10:16:03,221 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:03,222 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:03,222 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3b3cde69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:03,224 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:03,236 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:03,236 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:03,239 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930059 connected 2024-12-09T10:16:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-12-09T10:16:03,370 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:16:03,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3938): Client=jenkins//172.17.0.3 disable replication peer, id=1 2024-12-09T10:16:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=97, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-12-09T10:16:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-12-09T10:16:03,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-12-09T10:16:03,546 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:03,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=98 2024-12-09T10:16:03,546 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-12-09T10:16:03,547 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:03,547 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:03,547 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:03,547 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930059, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:03,548 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930059, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:03,661 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930059, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:03,661 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=98}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x43da105b to 127.0.0.1:64103 2024-12-09T10:16:03,661 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=98}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:03,679 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:03,680 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:03,680 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-09T10:16:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=98 2024-12-09T10:16:03,683 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for DISABLE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:03,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=98, resume processing ppid=97 2024-12-09T10:16:03,687 INFO [PEWorker-4 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-12-09T10:16:03,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=98, ppid=97, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 291 msec 2024-12-09T10:16:03,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=97, state=SUCCESS; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 312 msec 2024-12-09T10:16:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-12-09T10:16:03,700 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:16:03,705 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=99, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-12-09T10:16:03,718 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:03,719 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a4cd951 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6ae33621 2024-12-09T10:16:03,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:03,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:03,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:03,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:03,741 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:03,742 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:03,743 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4ad4df84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:03,744 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:03,771 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:03,771 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:03,787 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593005b connected 2024-12-09T10:16:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-12-09T10:16:03,895 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:03,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=100 2024-12-09T10:16:03,897 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:03,897 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:03,897 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:03,897 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:03,898 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593005b, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:03,898 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593005b, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:04,012 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593005b, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:04,012 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=100}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0a4cd951 to 127.0.0.1:64103 2024-12-09T10:16:04,013 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=100}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:04,013 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:04,025 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=100}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-12-09T10:16:04,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:04,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:04,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:04,061 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=100}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:04,061 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-09T10:16:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=100 2024-12-09T10:16:04,068 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:04,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=100, resume processing ppid=99 2024-12-09T10:16:04,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=100, ppid=99, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 335 msec 2024-12-09T10:16:04,100 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:04,101 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:04,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=99, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 396 msec 2024-12-09T10:16:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-12-09T10:16:04,340 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:04,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=101, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=101 2024-12-09T10:16:04,344 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:04,346 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=101, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:04,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=101, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-09T10:16:04,348 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=101 2024-12-09T10:16:04,449 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:04,450 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=102, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-12-09T10:16:04,452 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:04,454 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=102, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:04,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=102, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-09T10:16:04,464 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:04,522 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T10:16:04,524 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41504, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T10:16:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-12-09T10:16:04,560 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:04,573 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=221 (was 221), OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1279 (was 1279), ProcessCount=11 (was 11), AvailableMemoryMB=2576 (was 2674) 2024-12-09T10:16:04,588 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=221, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=1279, ProcessCount=11, AvailableMemoryMB=2575 2024-12-09T10:16:04,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=dummy_peer 2024-12-09T10:16:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=103, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=103 2024-12-09T10:16:04,591 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:04,593 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=103, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:04,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=103, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T10:16:04,594 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=103 2024-12-09T10:16:04,699 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-12-09T10:16:04,700 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=104, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-12-09T10:16:04,702 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:04,704 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=104, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:04,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=104, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T10:16:04,705 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:04,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-12-09T10:16:04,809 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:04,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:04,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=105, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:04,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=105 2024-12-09T10:16:04,811 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:04,812 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=105, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:04,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=105, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T10:16:04,814 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:04,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=105 2024-12-09T10:16:04,919 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:04,931 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=221 (was 221), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1279 (was 1279), ProcessCount=11 (was 11), AvailableMemoryMB=2567 (was 2575) 2024-12-09T10:16:04,941 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=221, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=1279, ProcessCount=11, AvailableMemoryMB=2567 2024-12-09T10:16:04,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:04,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=106, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:04,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-09T10:16:04,965 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:04,966 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:04,993 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:04,993 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593005c connected 2024-12-09T10:16:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-09T10:16:05,135 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593005c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:05,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-09T10:16:05,337 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:05,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=107 2024-12-09T10:16:05,338 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:05,365 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=107}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:05,396 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:05,550 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107 2024-12-09T10:16:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=107 2024-12-09T10:16:05,557 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:05,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=107, resume processing ppid=106 2024-12-09T10:16:05,560 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:05,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=107, ppid=106, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 373 msec 2024-12-09T10:16:05,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=106, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 617 msec 2024-12-09T10:16:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-09T10:16:05,570 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:16:05,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=108, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-09T10:16:05,596 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:05,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:05,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:05,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:05,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:05,614 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x08e0ff4f to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@47827329 2024-12-09T10:16:05,651 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:05,652 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:05,652 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@45ba7d0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:05,655 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-09T10:16:05,696 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:05,699 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593005e connected 2024-12-09T10:16:05,707 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:05,767 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:05,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=109 2024-12-09T10:16:05,773 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:05,773 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:05,773 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:05,775 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:05,775 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593005e, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:05,776 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593005e, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-09T10:16:05,900 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=109}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x08e0ff4f to 127.0.0.1:64103 2024-12-09T10:16:05,900 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=109}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:05,900 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:05,900 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593005e, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:05,914 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=109}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:05,938 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=109}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:05,938 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=109 2024-12-09T10:16:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=109 2024-12-09T10:16:05,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:05,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:05,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:05,941 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:05,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=109, resume processing ppid=108 2024-12-09T10:16:05,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=109, ppid=108, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 330 msec 2024-12-09T10:16:05,999 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:05,999 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:06,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=108, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 428 msec 2024-12-09T10:16:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-09T10:16:06,215 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:06,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=110, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-12-09T10:16:06,224 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:06,227 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=110, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:06,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=110, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=10 msec 2024-12-09T10:16:06,233 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-12-09T10:16:06,332 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:06,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=111, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=111 2024-12-09T10:16:06,335 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:06,337 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=111, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:06,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=111, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T10:16:06,339 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=111 2024-12-09T10:16:06,441 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:06,475 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=219 (was 221), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1279 (was 1279), ProcessCount=11 (was 11), AvailableMemoryMB=2412 (was 2567) 2024-12-09T10:16:06,499 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=219, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=1279, ProcessCount=11, AvailableMemoryMB=2410 2024-12-09T10:16:06,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-09T10:16:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] ipc.CallRunner(138): callId: 364 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.3:48524 deadline: 1733739426506, exception=java.io.IOException: Replication peer modification disabled 2024-12-09T10:16:06,508 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 2 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-12-09T10:16:06,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-09T10:16:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] ipc.CallRunner(138): callId: 365 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.3:48524 deadline: 1733739426620, exception=java.io.IOException: Replication peer modification disabled 2024-12-09T10:16:06,621 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 115 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-12-09T10:16:06,622 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-12-09T10:16:06.510Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-12-09T10:16:06.622Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-12-09T10:16:06,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=112, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-12-09T10:16:06,651 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:06,654 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=112, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:06,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=112, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=19 msec 2024-12-09T10:16:06,656 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-12-09T10:16:06,750 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:06,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=113, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=113 2024-12-09T10:16:06,759 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:06,779 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=113, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:06,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=113, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=27 msec 2024-12-09T10:16:06,783 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=113 2024-12-09T10:16:06,872 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:06,897 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=219 (was 219), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1279 (was 1279), ProcessCount=11 (was 11), AvailableMemoryMB=2393 (was 2410) 2024-12-09T10:16:06,915 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=219, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=1279, ProcessCount=11, AvailableMemoryMB=2392 2024-12-09T10:16:06,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=114, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-09T10:16:06,939 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:06,942 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:06,989 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:06,991 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593005f connected 2024-12-09T10:16:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-09T10:16:07,144 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593005f, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:07,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-09T10:16:07,378 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:07,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=115 2024-12-09T10:16:07,378 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:07,401 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=115}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:07,424 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-09T10:16:07,692 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:16:07,703 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=115 2024-12-09T10:16:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=115 2024-12-09T10:16:07,707 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:07,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=115, resume processing ppid=114 2024-12-09T10:16:07,710 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:07,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=115, ppid=114, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 482 msec 2024-12-09T10:16:07,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=114, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 794 msec 2024-12-09T10:16:07,745 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ec85148 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@265ddabb 2024-12-09T10:16:07,778 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:07,779 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:07,779 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@68ddc5c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:07,780 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:07,822 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:07,824 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930061 connected 2024-12-09T10:16:07,826 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:08,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T10:16:08,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-09T10:16:08,059 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:08,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:08,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=116, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-09T10:16:08,064 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:08,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:08,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:08,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:08,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-09T10:16:08,260 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:08,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=117 2024-12-09T10:16:08,261 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:08,261 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:08,261 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:08,261 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:08,261 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930061, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:08,261 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930061, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:08,375 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=117}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1ec85148 to 127.0.0.1:64103 2024-12-09T10:16:08,375 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930061, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:08,375 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=117}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:08,375 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-09T10:16:08,387 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=117}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:08,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:08,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:08,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:08,397 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=117}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:08,398 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=117 2024-12-09T10:16:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=117 2024-12-09T10:16:08,399 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:08,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=117, resume processing ppid=116 2024-12-09T10:16:08,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=117, ppid=116, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 293 msec 2024-12-09T10:16:08,414 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:08,414 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:08,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=116, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 353 msec 2024-12-09T10:16:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-09T10:16:08,689 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:08,689 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:08,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=118, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:08,691 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:08,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-12-09T10:16:08,693 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=118, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:08,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=118, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T10:16:08,694 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-12-09T10:16:08,810 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:08,834 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=219 (was 219), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1320 (was 1279) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2353 (was 2392) 2024-12-09T10:16:08,844 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=219, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=1320, ProcessCount=11, AvailableMemoryMB=2352 2024-12-09T10:16:08,845 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=119, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-12-09T10:16:08,874 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:08,875 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:08,924 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:08,931 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930062 connected 2024-12-09T10:16:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-12-09T10:16:09,060 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930062, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:09,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-12-09T10:16:09,234 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:09,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=120 2024-12-09T10:16:09,235 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:09,259 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=120}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:09,277 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:09,342 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-09T10:16:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=120 2024-12-09T10:16:09,344 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:09,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=120, resume processing ppid=119 2024-12-09T10:16:09,345 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:09,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=120, ppid=119, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec 2024-12-09T10:16:09,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 501 msec 2024-12-09T10:16:09,376 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5c96e697 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e8011a7 2024-12-09T10:16:09,389 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:09,390 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:09,390 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@63fab328, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:09,391 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:09,397 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:09,398 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:09,398 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930064 connected 2024-12-09T10:16:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-12-09T10:16:09,479 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:09,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:09,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-09T10:16:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=121, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-12-09T10:16:09,502 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:09,503 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:09,514 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:09,514 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930065 connected 2024-12-09T10:16:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-12-09T10:16:09,622 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930065, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:09,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:09,784 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:09,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=122 2024-12-09T10:16:09,785 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:09,786 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-09T10:16:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=122 2024-12-09T10:16:09,787 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:09,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=122, resume processing ppid=121 2024-12-09T10:16:09,788 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-09T10:16:09,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=122, ppid=121, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 156 msec 2024-12-09T10:16:09,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=121, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 307 msec 2024-12-09T10:16:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-12-09T10:16:09,800 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:09,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:09,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=123, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-12-09T10:16:09,803 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:09,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:09,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:09,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:09,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-12-09T10:16:09,967 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:09,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=124 2024-12-09T10:16:09,968 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:09,968 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:09,968 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:09,968 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:09,969 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930064, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:09,969 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930064, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:10,078 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930064, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:10,078 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=124}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5c96e697 to 127.0.0.1:64103 2024-12-09T10:16:10,078 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=124}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:10,078 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:10,094 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=124}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:10,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:10,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:10,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:10,106 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=124}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:10,106 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-09T10:16:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=124 2024-12-09T10:16:10,108 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:10,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=124, resume processing ppid=123 2024-12-09T10:16:10,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=124, ppid=123, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 293 msec 2024-12-09T10:16:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-12-09T10:16:10,120 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:10,120 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:10,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=123, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 319 msec 2024-12-09T10:16:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-12-09T10:16:10,429 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:10,430 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=125, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-12-09T10:16:10,431 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:10,433 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=125, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:10,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=125, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T10:16:10,434 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-12-09T10:16:10,540 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:10,540 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=126, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-12-09T10:16:10,542 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:10,544 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=126, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:10,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=126, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T10:16:10,546 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-12-09T10:16:10,650 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:10,663 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=219 (was 219), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1320 (was 1320), ProcessCount=11 (was 11), AvailableMemoryMB=2305 (was 2352) 2024-12-09T10:16:10,674 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=219, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=1320, ProcessCount=11, AvailableMemoryMB=2305 2024-12-09T10:16:10,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=127, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-12-09T10:16:10,698 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-12-09T10:16:10,850 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:10,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=128 2024-12-09T10:16:10,851 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:10,889 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:10,931 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-09T10:16:10,931 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(587): peerId=1, queueId=1 (queues=1) is replicating from cluster=0c6f32a7-e3f8-4a47-90f8-ff1b85bfb52d to cluster=0c6f32a7-e3f8-4a47-90f8-ff1b85bfb52d 2024-12-09T10:16:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=128 2024-12-09T10:16:10,932 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(343): peerId=1, starting shipping worker for walGroupId=de59e52b2fff%2C37621%2C1733739327100 2024-12-09T10:16:10,933 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:10,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=128, resume processing ppid=127 2024-12-09T10:16:10,934 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:10,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=128, ppid=127, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 235 msec 2024-12-09T10:16:10,935 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSourceWALReader(112): peerClusterZnode=1, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-12-09T10:16:10,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=127, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 258 msec 2024-12-09T10:16:10,937 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.wal-reader.de59e52b2fff%2C37621%2C1733739327100,1 {}] regionserver.WALEntryStream(260): Creating new reader hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/WALs/de59e52b2fff,37621,1733739327100/de59e52b2fff%2C37621%2C1733739327100.1733739331285, startPosition=0, beingWritten=true 2024-12-09T10:16:10,939 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.shipperde59e52b2fff%2C37621%2C1733739327100,1 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: de59e52b2fff%2C37621%2C1733739327100 2024-12-09T10:16:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-12-09T10:16:11,000 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:11,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=129, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=129 2024-12-09T10:16:11,003 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-12-09T10:16:11,004 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=129, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:11,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=129, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec 2024-12-09T10:16:11,006 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:11,010 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.wal-reader.de59e52b2fff%2C37621%2C1733739327100,1 {}] regionserver.ReplicationSourceWALReader(181): Read 0 WAL entries eligible for replication 2024-12-09T10:16:11,011 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.wal-reader.de59e52b2fff%2C37621%2C1733739327100,1 {}] regionserver.WALEntryStream(223): Reset reader hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/WALs/de59e52b2fff,37621,1733739327100/de59e52b2fff%2C37621%2C1733739327100.1733739331285 to pos 589, reset compression=false 2024-12-09T10:16:11,011 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.shipperde59e52b2fff%2C37621%2C1733739327100,1 {}] regionserver.ReplicationSourceShipper(110): Shipper from source 1 got entry batch from reader: WALEntryBatch [walEntries=[], lastWalPath=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/WALs/de59e52b2fff,37621,1733739327100/de59e52b2fff%2C37621%2C1733739327100.1733739331285, lastWalPosition=589, nbRowKeys=0, nbHFiles=0, heapSize=0, lastSeqIds={}, endOfFile=false,usedBufferSize=0] 2024-12-09T10:16:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=129 2024-12-09T10:16:11,110 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-12-09T10:16:11,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=130, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T10:16:11,116 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:11,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:11,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:11,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:11,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T10:16:11,283 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:11,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=131 2024-12-09T10:16:11,284 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:11,284 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:11,284 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T10:16:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T10:16:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T10:16:12,285 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.shipperde59e52b2fff%2C37621%2C1733739327100,1 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:313) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-12-09T10:16:12,285 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.wal-reader.de59e52b2fff%2C37621%2C1733739327100,1 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:131) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:166) ~[classes/:?] 2024-12-09T10:16:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T10:16:13,287 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSource(711): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.shipperde59e52b2fff%2C37621%2C1733739327100,1 terminated 2024-12-09T10:16:13,287 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:13,299 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=131}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:13,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:13,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:13,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:13,369 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=131}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:13,369 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=131 2024-12-09T10:16:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=131 2024-12-09T10:16:13,380 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:13,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=131, resume processing ppid=130 2024-12-09T10:16:13,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=131, ppid=130, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.2480 sec 2024-12-09T10:16:13,435 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:13,435 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:13,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=130, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.3250 sec 2024-12-09T10:16:13,888 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:16:14,132 INFO [master/de59e52b2fff:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T10:16:14,132 INFO [master/de59e52b2fff:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T10:16:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T10:16:15,285 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:15,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=132, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-12-09T10:16:15,287 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:15,289 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=132, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:15,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=132, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T10:16:15,290 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-12-09T10:16:15,399 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:15,418 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=219 (was 219), OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1311 (was 1320), ProcessCount=11 (was 11), AvailableMemoryMB=2823 (was 2305) - AvailableMemoryMB LEAK? - 2024-12-09T10:16:15,432 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=219, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=1311, ProcessCount=11, AvailableMemoryMB=2820 2024-12-09T10:16:15,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=133, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-12-09T10:16:15,459 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:15,460 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:15,511 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:15,511 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930066 connected 2024-12-09T10:16:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-12-09T10:16:15,631 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930066, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:15,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-12-09T10:16:15,810 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:15,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=134 2024-12-09T10:16:15,812 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:15,852 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=134}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:15,896 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:15,967 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-09T10:16:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=134 2024-12-09T10:16:15,974 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:15,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=134, resume processing ppid=133 2024-12-09T10:16:15,976 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:15,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=134, ppid=133, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 318 msec 2024-12-09T10:16:15,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=133, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 542 msec 2024-12-09T10:16:16,037 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2d2280a3 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2fe1b984 2024-12-09T10:16:16,053 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:16,054 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:16,054 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@748331e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:16,055 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:16,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-12-09T10:16:16,062 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:16,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:16:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=135, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-12-09T10:16:16,071 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:16,071 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:16,087 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930068 connected 2024-12-09T10:16:16,126 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:16,127 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-12-09T10:16:16,183 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:16,312 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:16,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-12-09T10:16:16,482 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:16,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=136 2024-12-09T10:16:16,484 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:16,485 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:16,485 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:16,485 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:16,485 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930068, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:16,486 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930068, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:16,594 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930068, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:16,594 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=136}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2d2280a3 to 127.0.0.1:64103 2024-12-09T10:16:16,595 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=136}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:16,611 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:16,611 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:16,615 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-09T10:16:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=136 2024-12-09T10:16:16,622 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:16,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=136, resume processing ppid=135 2024-12-09T10:16:16,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=136, ppid=135, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 294 msec 2024-12-09T10:16:16,625 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:16:16,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=135, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 561 msec 2024-12-09T10:16:16,656 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x30bd672b to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@49d1f0b9 2024-12-09T10:16:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-12-09T10:16:16,699 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:16,700 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:16,701 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7210b5e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:16,702 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:16,703 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:16,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:16,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:16,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:16,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T10:16:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=137, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-12-09T10:16:16,740 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:16,747 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:16,753 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:16,754 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:16,755 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593006b connected 2024-12-09T10:16:16,781 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:16,785 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593006c connected 2024-12-09T10:16:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-12-09T10:16:16,911 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593006c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:16,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-12-09T10:16:17,077 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:17,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=138 2024-12-09T10:16:17,079 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:17,082 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:17,083 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:17,083 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:17,083 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593006b, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:17,083 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593006b, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:17,197 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=138}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x30bd672b to 127.0.0.1:64103 2024-12-09T10:16:17,198 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=138}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:17,199 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593006b, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:17,220 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:17,221 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:17,221 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-09T10:16:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=138 2024-12-09T10:16:17,236 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:17,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=138, resume processing ppid=137 2024-12-09T10:16:17,244 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T10:16:17,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=138, ppid=137, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 313 msec 2024-12-09T10:16:17,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=137, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 520 msec 2024-12-09T10:16:17,262 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x32eeaf80 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@21a1b631 2024-12-09T10:16:17,304 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:17,305 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:17,310 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6edd806a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:17,313 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:17,322 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:17,323 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593006e connected 2024-12-09T10:16:17,327 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-12-09T10:16:17,369 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:17,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:17,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:17,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:17,378 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:17,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-09T10:16:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=139, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-12-09T10:16:17,419 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:17,420 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:17,431 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:17,432 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593006f connected 2024-12-09T10:16:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-12-09T10:16:17,548 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593006f, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-12-09T10:16:17,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:17,915 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:17,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=140 2024-12-09T10:16:17,916 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:17,924 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:17,924 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:17,925 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:17,925 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593006e, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:17,925 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593006e, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-12-09T10:16:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T10:16:18,072 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593006e, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:18,075 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=140}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x32eeaf80 to 127.0.0.1:64103 2024-12-09T10:16:18,075 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=140}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:18,184 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:18,184 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:18,195 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-12-09T10:16:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=140 2024-12-09T10:16:18,215 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:18,222 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-09T10:16:18,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=140, resume processing ppid=139 2024-12-09T10:16:18,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=140, ppid=139, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 460 msec 2024-12-09T10:16:18,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=139, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 842 msec 2024-12-09T10:16:18,237 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3829ce18 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2fbd309b 2024-12-09T10:16:18,267 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:18,268 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:18,268 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@545ebcce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:18,269 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:18,311 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:18,311 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:18,314 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930071 connected 2024-12-09T10:16:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-12-09T10:16:18,519 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:18,520 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:18,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:18,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:18,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:16:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=141, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-12-09T10:16:18,544 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:18,545 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:18,577 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:18,579 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930072 connected 2024-12-09T10:16:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-12-09T10:16:18,694 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930072, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:18,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:18,789 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:16:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-12-09T10:16:18,858 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:18,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=142 2024-12-09T10:16:18,859 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:18,860 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:18,860 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:18,860 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:18,860 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930071, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:18,861 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930071, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:18,969 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930071, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:18,969 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=142}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3829ce18 to 127.0.0.1:64103 2024-12-09T10:16:18,969 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=142}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:18,990 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:18,990 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:18,991 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-12-09T10:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=142 2024-12-09T10:16:19,007 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:19,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=142, resume processing ppid=141 2024-12-09T10:16:19,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=142, ppid=141, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 301 msec 2024-12-09T10:16:19,019 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:16:19,031 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=141, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 497 msec 2024-12-09T10:16:19,048 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4808da1a to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6dca02 2024-12-09T10:16:19,104 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:19,105 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:19,105 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@348430cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:19,106 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:19,119 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:19,120 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:19,126 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930074 connected 2024-12-09T10:16:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-12-09T10:16:19,154 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:19,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:19,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:19,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-09T10:16:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=143, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-12-09T10:16:19,181 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:19,182 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:19,197 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:19,203 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930075 connected 2024-12-09T10:16:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-12-09T10:16:19,319 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930075, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:19,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-12-09T10:16:19,487 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:19,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=144 2024-12-09T10:16:19,491 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:19,508 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:19,508 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:19,508 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:19,508 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930074, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:19,509 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930074, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:19,620 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930074, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:19,620 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=144}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4808da1a to 127.0.0.1:64103 2024-12-09T10:16:19,620 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=144}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:19,637 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:19,637 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:19,637 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-12-09T10:16:19,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=144 2024-12-09T10:16:19,641 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:19,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=144, resume processing ppid=143 2024-12-09T10:16:19,644 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-09T10:16:19,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=144, ppid=143, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 310 msec 2024-12-09T10:16:19,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=143, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 487 msec 2024-12-09T10:16:19,671 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x410a1309 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a5c5db6 2024-12-09T10:16:19,687 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:19,688 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:19,688 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3f986bb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:19,689 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:19,702 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:19,703 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930077 connected 2024-12-09T10:16:19,703 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-12-09T10:16:19,799 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:19,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:19,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:16:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=145, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-12-09T10:16:19,833 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:19,833 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:19,863 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:19,868 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930078 connected 2024-12-09T10:16:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-12-09T10:16:19,990 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930078, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:20,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-12-09T10:16:20,159 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:20,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=146 2024-12-09T10:16:20,160 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:20,162 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:20,162 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:20,162 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:20,162 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930077, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:20,162 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930077, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:20,285 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=146}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x410a1309 to 127.0.0.1:64103 2024-12-09T10:16:20,285 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930077, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:20,285 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=146}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:20,306 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:20,307 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:20,307 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-12-09T10:16:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=146 2024-12-09T10:16:20,319 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:20,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=146, resume processing ppid=145 2024-12-09T10:16:20,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=146, ppid=145, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 316 msec 2024-12-09T10:16:20,322 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:16:20,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=145, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 516 msec 2024-12-09T10:16:20,350 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2cba3d7a to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@68ab12d3 2024-12-09T10:16:20,404 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:20,404 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:20,404 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2971d0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:20,405 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:20,425 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:20,429 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593007a connected 2024-12-09T10:16:20,429 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-12-09T10:16:20,452 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:20,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:20,463 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:20,463 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=147, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=148, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-09T10:16:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-12-09T10:16:20,495 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:20,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:20,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:20,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:20,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-09T10:16:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-12-09T10:16:20,715 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:20,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=149 2024-12-09T10:16:20,715 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:20,716 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:20,716 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:20,716 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:20,716 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593007a, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:20,716 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593007a, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-09T10:16:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-12-09T10:16:20,830 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593007a, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:20,830 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=149}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2cba3d7a to 127.0.0.1:64103 2024-12-09T10:16:20,831 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=149}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:20,831 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:20,839 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=149}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:20,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:20,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:20,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:20,856 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=149}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:20,856 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-09T10:16:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=149 2024-12-09T10:16:20,879 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:20,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=149, resume processing ppid=147 2024-12-09T10:16:20,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=149, ppid=147, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 319 msec 2024-12-09T10:16:20,921 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:20,921 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:20,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=147, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 458 msec 2024-12-09T10:16:20,927 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:20,928 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=148, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:20,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=148, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=466 msec 2024-12-09T10:16:20,930 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-12-09T10:16:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-09T10:16:21,099 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:21,099 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:21,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=150, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=150 2024-12-09T10:16:21,116 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:21,117 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=150, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:21,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=150, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=10 msec 2024-12-09T10:16:21,118 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=150 2024-12-09T10:16:21,223 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:21,254 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=218 (was 219), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1286 (was 1311), ProcessCount=11 (was 11), AvailableMemoryMB=2299 (was 2820) 2024-12-09T10:16:21,268 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=218, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=1286, ProcessCount=11, AvailableMemoryMB=2299 2024-12-09T10:16:21,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=151, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-12-09T10:16:21,301 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:21,302 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:21,314 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:21,317 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593007b connected 2024-12-09T10:16:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-12-09T10:16:21,436 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593007b, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:21,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-12-09T10:16:21,657 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:21,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=152 2024-12-09T10:16:21,658 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:21,677 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=152}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:21,697 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:21,759 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152 2024-12-09T10:16:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=152 2024-12-09T10:16:21,765 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:21,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=152, resume processing ppid=151 2024-12-09T10:16:21,780 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:21,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=152, ppid=151, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 268 msec 2024-12-09T10:16:21,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=151, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 509 msec 2024-12-09T10:16:21,809 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d9c0840 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3febfc23 2024-12-09T10:16:21,834 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:21,834 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:21,835 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@301856ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:21,836 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:21,847 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:21,848 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593007d connected 2024-12-09T10:16:21,848 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-12-09T10:16:21,903 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:21,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:16:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=153, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-12-09T10:16:21,936 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:21,941 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:21,958 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:21,961 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593007e connected 2024-12-09T10:16:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-12-09T10:16:22,078 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593007e, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:22,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-12-09T10:16:22,256 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:22,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=154 2024-12-09T10:16:22,259 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:22,280 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:22,280 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:22,280 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:22,280 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593007d, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:22,281 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593007d, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:22,395 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=154}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7d9c0840 to 127.0.0.1:64103 2024-12-09T10:16:22,396 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=154}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:22,399 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593007d, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:22,425 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:22,425 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:22,431 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=154 2024-12-09T10:16:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=154 2024-12-09T10:16:22,440 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:22,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=154, resume processing ppid=153 2024-12-09T10:16:22,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=154, ppid=153, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 338 msec 2024-12-09T10:16:22,452 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:16:22,459 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=153, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 547 msec 2024-12-09T10:16:22,492 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1d51da8f to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@377dc2c3 2024-12-09T10:16:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-12-09T10:16:22,544 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:22,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:22,545 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:22,546 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:22,546 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1f08b69b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:22,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T10:16:22,548 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=155, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-09T10:16:22,574 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:22,575 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:22,579 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:22,587 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930080 connected 2024-12-09T10:16:22,589 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:22,602 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:22,607 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930081 connected 2024-12-09T10:16:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-09T10:16:22,725 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930081, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-09T10:16:22,961 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:23,119 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:23,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=156 2024-12-09T10:16:23,122 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:23,141 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:23,141 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:23,141 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:23,141 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930080, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:23,142 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930080, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-09T10:16:23,261 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=156}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1d51da8f to 127.0.0.1:64103 2024-12-09T10:16:23,261 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930080, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:23,262 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=156}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:23,285 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:23,286 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:23,288 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=156 2024-12-09T10:16:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=156 2024-12-09T10:16:23,294 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:23,296 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=156, resume processing ppid=155 2024-12-09T10:16:23,296 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=156, ppid=155, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 333 msec 2024-12-09T10:16:23,296 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T10:16:23,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=155, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 749 msec 2024-12-09T10:16:23,342 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x002207af to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1e07887d 2024-12-09T10:16:23,392 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:23,392 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:23,393 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3ba2515d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:23,400 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:23,433 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:23,436 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930083 connected 2024-12-09T10:16:23,443 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-09T10:16:23,689 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:23,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:23,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-12-09T10:16:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=157, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-09T10:16:23,720 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:23,721 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:23,751 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:23,763 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930084 connected 2024-12-09T10:16:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-09T10:16:23,912 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930084, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:23,912 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:23,919 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=157, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:23,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=157, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=223 msec 2024-12-09T10:16:23,925 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-09T10:16:24,020 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-12-09T10:16:24,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:24,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-09T10:16:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=158, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-12-09T10:16:24,057 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:24,058 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:24,079 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-12-09T10:16:24,193 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:24,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-12-09T10:16:24,372 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:24,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=159 2024-12-09T10:16:24,379 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:24,390 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:24,390 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:24,391 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:24,391 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930083, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:24,391 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930083, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:24,512 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930083, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:24,515 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=159}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x002207af to 127.0.0.1:64103 2024-12-09T10:16:24,515 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=159}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:24,533 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:24,534 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:24,534 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-09T10:16:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=159 2024-12-09T10:16:24,549 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:24,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=159, resume processing ppid=158 2024-12-09T10:16:24,571 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-09T10:16:24,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=159, ppid=158, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 332 msec 2024-12-09T10:16:24,573 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6c397e42 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3254e403 2024-12-09T10:16:24,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=158, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 547 msec 2024-12-09T10:16:24,599 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:24,600 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:24,600 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5b2763f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:24,601 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:24,619 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:24,619 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:24,627 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930087 connected 2024-12-09T10:16:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-12-09T10:16:24,649 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:24,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:24,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-09T10:16:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=160, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=160 2024-12-09T10:16:24,686 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:24,687 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:24,711 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=160 2024-12-09T10:16:24,811 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:24,815 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:24,817 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=160, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:24,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=160, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=165 msec 2024-12-09T10:16:24,818 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:24,922 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T10:16:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=160 2024-12-09T10:16:24,981 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-12-09T10:16:24,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=161, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-09T10:16:25,007 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:25,033 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:25,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:25,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:25,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-09T10:16:25,185 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:25,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=162 2024-12-09T10:16:25,187 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:25,187 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:25,187 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:25,187 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:25,188 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930087, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:25,188 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930087, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:25,303 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930087, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:25,303 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=162}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6c397e42 to 127.0.0.1:64103 2024-12-09T10:16:25,304 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=162}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:25,304 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-09T10:16:25,319 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=162}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:25,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:25,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:25,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:25,330 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=162}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:25,330 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=162 2024-12-09T10:16:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=162 2024-12-09T10:16:25,333 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:25,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=162, resume processing ppid=161 2024-12-09T10:16:25,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=162, ppid=161, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 300 msec 2024-12-09T10:16:25,349 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:25,349 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:25,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=161, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 367 msec 2024-12-09T10:16:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-09T10:16:25,627 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:25,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=163, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:25,637 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:25,638 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=163, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:25,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=163, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=10 msec 2024-12-09T10:16:25,639 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-12-09T10:16:25,644 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:25,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=164, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=164 2024-12-09T10:16:25,647 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:25,648 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=164, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:25,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=164, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T10:16:25,650 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=164 2024-12-09T10:16:25,755 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:25,777 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=215 (was 218), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1327 (was 1286) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2145 (was 2299) 2024-12-09T10:16:25,794 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=215, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=1327, ProcessCount=11, AvailableMemoryMB=2146 2024-12-09T10:16:25,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=165, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-09T10:16:25,823 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:25,824 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:25,851 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:25,857 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930089 connected 2024-12-09T10:16:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-09T10:16:26,008 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930089, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:26,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-09T10:16:26,207 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:26,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=166 2024-12-09T10:16:26,208 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=166}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:26,251 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=166}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:26,278 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=166}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:26,333 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:16:26,375 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=166}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=166 2024-12-09T10:16:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=166 2024-12-09T10:16:26,384 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:26,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=166, resume processing ppid=165 2024-12-09T10:16:26,387 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:26,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=166, ppid=165, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 336 msec 2024-12-09T10:16:26,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=165, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 592 msec 2024-12-09T10:16:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-09T10:16:26,432 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:26,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=167, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-09T10:16:26,454 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0b2fb0ed to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7af7ce76 2024-12-09T10:16:26,472 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:26,473 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:26,503 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:26,505 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:26,505 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6f3475b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:26,506 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:26,515 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:26,519 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593008b connected 2024-12-09T10:16:26,531 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:26,543 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:26,547 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593008c connected 2024-12-09T10:16:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-09T10:16:26,641 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593008b, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:26,641 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:26,644 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=167, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:26,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=167, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=208 msec 2024-12-09T10:16:26,645 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-09T10:16:26,759 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-12-09T10:16:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:16:26,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=168, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=168 2024-12-09T10:16:26,770 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:26,771 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=168, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:26,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=168, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=11 msec 2024-12-09T10:16:26,772 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=168 2024-12-09T10:16:26,879 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:16:26,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=127.0.0.1:64103:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=169, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-12-09T10:16:26,912 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:26,913 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:26,927 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:26,931 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593008d connected 2024-12-09T10:16:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-12-09T10:16:27,039 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593008d, quorum=127.0.0.1:64103, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:27,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-12-09T10:16:27,259 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:27,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=170 2024-12-09T10:16:27,261 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=ADD_PEER 2024-12-09T10:16:27,294 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=170}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:27,315 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSource(229): queueId=2, ReplicationSource: 2, currentBandwidth=0 2024-12-09T10:16:27,339 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=170 2024-12-09T10:16:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=170 2024-12-09T10:16:27,342 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:27,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=170, resume processing ppid=169 2024-12-09T10:16:27,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=170, ppid=169, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 242 msec 2024-12-09T10:16:27,352 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 2, config clusterKey=127.0.0.1:64103:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:27,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=169, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 472 msec 2024-12-09T10:16:27,398 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1f935f5a to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e3fcad5 2024-12-09T10:16:27,418 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:27,419 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:27,419 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@43ce5508, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:27,420 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 2 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:27,438 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 20x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:27,439 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 2-0x100be888593008f connected 2024-12-09T10:16:27,439 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] regionserver.ReplicationSource(569): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-12-09T10:16:27,509 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-12-09T10:16:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:16:27,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=171, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-12-09T10:16:27,514 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:27,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:27,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:27,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:27,534 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-12-09T10:16:27,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-12-09T10:16:27,689 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:27,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=172 2024-12-09T10:16:27,690 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:27,690 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:27,690 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:27,690 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:27,690 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593008c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:27,691 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593008c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:27,797 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593008c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:27,797 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=172}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0b2fb0ed to 127.0.0.1:64103 2024-12-09T10:16:27,797 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=172}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:27,798 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:27,805 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=172}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:27,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:27,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:27,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:27,814 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=172}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:27,814 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=172 2024-12-09T10:16:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=172 2024-12-09T10:16:27,816 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:27,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=172, resume processing ppid=171 2024-12-09T10:16:27,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=172, ppid=171, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 280 msec 2024-12-09T10:16:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-12-09T10:16:27,835 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:27,835 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:27,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=171, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 325 msec 2024-12-09T10:16:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-12-09T10:16:28,139 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:16:28,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=173, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=173 2024-12-09T10:16:28,156 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-12-09T10:16:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T10:16:28,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-12-09T10:16:28,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-09T10:16:28,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-09T10:16:28,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=173 2024-12-09T10:16:28,322 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:28,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=174 2024-12-09T10:16:28,323 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-12-09T10:16:28,323 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 2: 0 2024-12-09T10:16:28,323 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.ReplicationSource(657): peerId=2, Closing source 2 because: Replication stream was removed by a user 2024-12-09T10:16:28,323 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] regionserver.ReplicationSource(508): peerId=2, Interrupted while sleeping between retries 2024-12-09T10:16:28,323 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] zookeeper.ZKUtil(165): connection to cluster: 2-0x100be888593008f, quorum=127.0.0.1:64103, baseZNode=/hbase-test2 Unable to set watcher on znode (/hbase-test2/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:28,323 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2 {}] zookeeper.ZKWatcher(734): connection to cluster: 2-0x100be888593008f, quorum=127.0.0.1:64103, baseZNode=/hbase-test2 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:28,430 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 2-0x100be888593008f, quorum=127.0.0.1:64103, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:28,430 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=174}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1f935f5a to 127.0.0.1:64103 2024-12-09T10:16:28,430 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=174}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:28,430 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.ReplicationSourceManager(430): Done with the queue 2 2024-12-09T10:16:28,446 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=174}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/2/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=173 2024-12-09T10:16:28,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/2/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:28,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/2 2024-12-09T10:16:28,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/2 2024-12-09T10:16:28,460 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=174}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-12-09T10:16:28,461 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=174 2024-12-09T10:16:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=174 2024-12-09T10:16:28,463 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:28,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=174, resume processing ppid=173 2024-12-09T10:16:28,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=174, ppid=173, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 293 msec 2024-12-09T10:16:28,472 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-12-09T10:16:28,472 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 2 2024-12-09T10:16:28,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=173, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 331 msec 2024-12-09T10:16:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=173 2024-12-09T10:16:28,772 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-12-09T10:16:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:16:28,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=175, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=175 2024-12-09T10:16:28,796 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:28,800 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=175, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:28,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=175, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=17 msec 2024-12-09T10:16:28,804 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=175 2024-12-09T10:16:28,900 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:28,907 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=176, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:28,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=176 2024-12-09T10:16:28,916 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:28,920 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=176, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:28,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=176, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=12 msec 2024-12-09T10:16:28,924 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=176 2024-12-09T10:16:29,024 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:29,059 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=215 (was 215), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1373 (was 1327) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3272 (was 2146) - AvailableMemoryMB LEAK? - 2024-12-09T10:16:29,070 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=215, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=1373, ProcessCount=11, AvailableMemoryMB=3271 2024-12-09T10:16:29,071 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=177, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=177 2024-12-09T10:16:29,085 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] ... 12 more 2024-12-09T10:16:29,093 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=177, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:29,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=177, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=23 msec 2024-12-09T10:16:29,104 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=177 2024-12-09T10:16:29,192 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-12-09T10:16:29,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=178, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=178 2024-12-09T10:16:29,204 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:29,215 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=178, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:29,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=178, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=20 msec 2024-12-09T10:16:29,220 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=178 2024-12-09T10:16:29,311 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:29,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=179, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=179 2024-12-09T10:16:29,325 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:29,326 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=179, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:29,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=179, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=12 msec 2024-12-09T10:16:29,328 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=179 2024-12-09T10:16:29,429 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:29,444 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=215 (was 215), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1373 (was 1373), ProcessCount=11 (was 11), AvailableMemoryMB=3228 (was 3271) 2024-12-09T10:16:29,453 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=215, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=1373, ProcessCount=11, AvailableMemoryMB=3227 2024-12-09T10:16:29,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=180, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=180 2024-12-09T10:16:29,456 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-12-09T10:16:29,457 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=180, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:29,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=180, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec 2024-12-09T10:16:29,457 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=180 2024-12-09T10:16:29,559 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-12-09T10:16:29,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=181, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:29,562 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=181 2024-12-09T10:16:29,564 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=181, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:29,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=181, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T10:16:29,565 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=181 2024-12-09T10:16:29,670 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:29,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=182, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=182 2024-12-09T10:16:29,672 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:29,672 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=182, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:29,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=182, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T10:16:29,673 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=182 2024-12-09T10:16:29,780 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:29,809 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=215 (was 215), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1373 (was 1373), ProcessCount=11 (was 11), AvailableMemoryMB=3191 (was 3227) 2024-12-09T10:16:29,820 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=215, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=1373, ProcessCount=11, AvailableMemoryMB=3190 2024-12-09T10:16:29,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=183, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-12-09T10:16:29,861 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:29,862 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:29,881 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:29,887 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930090 connected 2024-12-09T10:16:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-12-09T10:16:29,997 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930090, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:30,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=184, ppid=183, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-12-09T10:16:30,195 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:30,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=184 2024-12-09T10:16:30,196 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:30,221 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=184}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:30,241 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:30,319 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=184 2024-12-09T10:16:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=184 2024-12-09T10:16:30,322 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:30,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=184, resume processing ppid=183 2024-12-09T10:16:30,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=184, ppid=183, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 287 msec 2024-12-09T10:16:30,325 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:30,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=183, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 502 msec 2024-12-09T10:16:30,357 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6bda4887 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@40ba261e 2024-12-09T10:16:30,384 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:30,384 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:30,385 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6e5c293c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:30,385 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:30,397 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:30,397 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:30,397 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930092 connected 2024-12-09T10:16:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-12-09T10:16:30,450 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:30,450 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:16:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=185, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-12-09T10:16:30,474 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:30,475 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:30,487 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:30,488 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930093 connected 2024-12-09T10:16:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-12-09T10:16:30,630 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930093, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:30,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=186, ppid=185, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-12-09T10:16:30,804 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:30,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=186 2024-12-09T10:16:30,805 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:30,806 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:30,806 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:30,806 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:30,806 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930092, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:30,806 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930092, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:30,913 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930092, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:30,913 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=186}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6bda4887 to 127.0.0.1:64103 2024-12-09T10:16:30,913 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=186}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:30,928 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:30,928 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:30,928 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=186 2024-12-09T10:16:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=186 2024-12-09T10:16:30,930 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:30,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=186, resume processing ppid=185 2024-12-09T10:16:30,932 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:16:30,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=186, ppid=185, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 278 msec 2024-12-09T10:16:30,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=185, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 482 msec 2024-12-09T10:16:30,959 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1c216ca4 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42b0621f 2024-12-09T10:16:30,979 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:30,979 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:30,980 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1b7107c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:30,980 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:30,993 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:30,994 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:30,995 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930095 connected 2024-12-09T10:16:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-12-09T10:16:31,089 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:31,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:31,090 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-09T10:16:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=187, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-12-09T10:16:31,111 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:31,112 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:31,121 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:31,127 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930096 connected 2024-12-09T10:16:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-12-09T10:16:31,238 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930096, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:31,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=188, ppid=187, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:31,399 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:31,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=188 2024-12-09T10:16:31,404 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:31,408 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:31,408 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:31,408 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-12-09T10:16:31,408 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930095, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:31,408 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930095, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:31,520 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=188}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1c216ca4 to 127.0.0.1:64103 2024-12-09T10:16:31,520 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=188}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:31,521 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930095, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:31,545 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:31,545 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:31,552 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=188 2024-12-09T10:16:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=188 2024-12-09T10:16:31,566 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:31,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=188, resume processing ppid=187 2024-12-09T10:16:31,573 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-09T10:16:31,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=188, ppid=187, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 319 msec 2024-12-09T10:16:31,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=187, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 483 msec 2024-12-09T10:16:31,618 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5316d99e to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6fa26eab 2024-12-09T10:16:31,671 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:31,672 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:31,672 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6b67cda4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:31,673 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:31,689 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:31,690 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:31,699 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be8885930098 connected 2024-12-09T10:16:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-12-09T10:16:31,720 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:31,720 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:31,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:31,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T10:16:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=189, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-12-09T10:16:31,763 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:31,764 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:31,787 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:31,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930099 connected 2024-12-09T10:16:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-12-09T10:16:31,906 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be8885930099, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:31,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=190, ppid=189, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-12-09T10:16:32,115 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:32,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=190 2024-12-09T10:16:32,120 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:32,125 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:32,125 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:32,125 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:32,126 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be8885930098, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:32,126 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be8885930098, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:32,279 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be8885930098, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:32,280 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=190}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5316d99e to 127.0.0.1:64103 2024-12-09T10:16:32,280 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=190}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:32,311 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:32,311 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:32,315 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=190 2024-12-09T10:16:32,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=190 2024-12-09T10:16:32,318 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:32,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=190, resume processing ppid=189 2024-12-09T10:16:32,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=190, ppid=189, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 365 msec 2024-12-09T10:16:32,323 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T10:16:32,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=189, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 601 msec 2024-12-09T10:16:32,358 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d71ee3d to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@347e0009 2024-12-09T10:16:32,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-12-09T10:16:32,360 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:32,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:32,361 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=191, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=191 2024-12-09T10:16:32,372 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:32,372 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:32,373 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:32,373 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@42d58e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:32,375 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:32,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:32,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:32,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:32,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=192, ppid=191, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:32,405 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:32,411 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:32,415 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593009b connected 2024-12-09T10:16:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=191 2024-12-09T10:16:32,535 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:32,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=192 2024-12-09T10:16:32,536 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:32,536 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:32,536 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:32,536 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:32,536 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593009b, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:32,536 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593009b, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:32,647 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=192}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3d71ee3d to 127.0.0.1:64103 2024-12-09T10:16:32,647 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=192}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:32,647 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:32,647 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593009b, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:32,670 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=192}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=191 2024-12-09T10:16:32,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:32,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:32,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:32,684 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=192}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:32,684 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=192 2024-12-09T10:16:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=192 2024-12-09T10:16:32,694 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:32,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=192, resume processing ppid=191 2024-12-09T10:16:32,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=192, ppid=191, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 314 msec 2024-12-09T10:16:32,718 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:32,718 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:32,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=191, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 357 msec 2024-12-09T10:16:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=191 2024-12-09T10:16:32,989 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:32,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=193, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=193 2024-12-09T10:16:32,992 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:32,993 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=193, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:32,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=193, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T10:16:32,994 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=193 2024-12-09T10:16:33,102 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:33,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:33,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=194, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=194 2024-12-09T10:16:33,106 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:33,108 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=194, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:33,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=194, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-09T10:16:33,110 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=194 2024-12-09T10:16:33,209 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:33,224 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=216 (was 215) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1375 (was 1373) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2894 (was 3190) 2024-12-09T10:16:33,237 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=216, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=1375, ProcessCount=11, AvailableMemoryMB=2893 2024-12-09T10:16:33,238 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=195, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:33,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-12-09T10:16:33,260 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:33,260 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:33,297 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:33,297 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593009c connected 2024-12-09T10:16:33,299 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:16:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-12-09T10:16:33,372 DEBUG [master/de59e52b2fff:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region cb29027c56d4cda2e8d456ca713f729b changed from -1.0 to 0.0, refreshing cache 2024-12-09T10:16:33,412 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be888593009c, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:33,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=196, ppid=195, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-12-09T10:16:33,600 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:33,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=196 2024-12-09T10:16:33,600 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:33,627 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=196}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:33,649 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:33,719 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=196 2024-12-09T10:16:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=196 2024-12-09T10:16:33,722 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:33,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=196, resume processing ppid=195 2024-12-09T10:16:33,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=196, ppid=195, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 282 msec 2024-12-09T10:16:33,725 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:33,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=195, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 487 msec 2024-12-09T10:16:33,799 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7384a340 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6aa36628 2024-12-09T10:16:33,867 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:33,867 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:33,868 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4488d08b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:33,869 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-12-09T10:16:33,872 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:33,876 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:16:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=197, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-12-09T10:16:33,907 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:33,908 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:33,909 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:33,911 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be888593009e connected 2024-12-09T10:16:33,912 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:33,940 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:33,941 ERROR [PEWorker-5-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$361/0x00007f9d48914be0@170e1e4f rejected from java.util.concurrent.ThreadPoolExecutor@520c82ca[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.PendingWatcher.process(PendingWatcher.java:38) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-09T10:16:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-12-09T10:16:34,083 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:34,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=198, ppid=197, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-12-09T10:16:34,313 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:34,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=198 2024-12-09T10:16:34,314 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:34,315 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:34,315 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:34,315 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:34,315 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be888593009e, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:34,316 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be888593009e, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:34,439 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be888593009e, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:34,439 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=198}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7384a340 to 127.0.0.1:64103 2024-12-09T10:16:34,439 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=198}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:34,464 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:34,464 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:34,465 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=198 2024-12-09T10:16:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=198 2024-12-09T10:16:34,467 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:34,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=198, resume processing ppid=197 2024-12-09T10:16:34,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=198, ppid=197, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 307 msec 2024-12-09T10:16:34,472 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T10:16:34,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=197, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 597 msec 2024-12-09T10:16:34,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-12-09T10:16:34,509 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:34,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:34,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T10:16:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=199, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-12-09T10:16:34,521 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x10a258b5 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@79446452 2024-12-09T10:16:34,542 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:34,543 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:34,560 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:34,561 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:34,562 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@370dae46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:34,563 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:34,575 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:34,578 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300a1 connected 2024-12-09T10:16:34,611 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:34,611 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be88859300a2 connected 2024-12-09T10:16:34,615 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-12-09T10:16:34,691 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300a1, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:34,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=200, ppid=199, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-12-09T10:16:34,876 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:34,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=200 2024-12-09T10:16:34,876 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:34,883 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:34,883 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:34,883 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:34,883 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be88859300a2, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:34,884 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be88859300a2, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:34,997 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be88859300a2, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:34,999 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=200}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x10a258b5 to 127.0.0.1:64103 2024-12-09T10:16:34,999 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=200}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:35,032 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:35,032 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:35,032 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=200 2024-12-09T10:16:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=200 2024-12-09T10:16:35,037 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:35,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=200, resume processing ppid=199 2024-12-09T10:16:35,038 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T10:16:35,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=200, ppid=199, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 314 msec 2024-12-09T10:16:35,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=199, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 526 msec 2024-12-09T10:16:35,074 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x748626a3 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e116caa 2024-12-09T10:16:35,089 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:35,090 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:35,090 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@9cf8c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:35,091 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:35,109 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:35,111 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:35,119 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be88859300a4 connected 2024-12-09T10:16:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-12-09T10:16:35,150 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:35,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:35,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:35,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T10:16:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=201, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-12-09T10:16:35,184 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:35,184 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:35,209 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-12-09T10:16:35,321 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:35,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=202, ppid=201, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-12-09T10:16:35,503 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:35,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=202 2024-12-09T10:16:35,508 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:35,519 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:35,519 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:35,523 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:35,523 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be88859300a4, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:35,524 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be88859300a4, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:35,640 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be88859300a4, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:35,641 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=202}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x748626a3 to 127.0.0.1:64103 2024-12-09T10:16:35,641 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=202}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:35,659 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:35,659 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:35,660 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=202 2024-12-09T10:16:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=202 2024-12-09T10:16:35,662 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:35,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=202, resume processing ppid=201 2024-12-09T10:16:35,665 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T10:16:35,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=202, ppid=201, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 312 msec 2024-12-09T10:16:35,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=201, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 509 msec 2024-12-09T10:16:35,697 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d007003 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@bc0ba94 2024-12-09T10:16:35,717 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:35,718 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:35,718 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5a316d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:35,719 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:35,743 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:35,744 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:35,744 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be88859300a7 connected 2024-12-09T10:16:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-12-09T10:16:35,789 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:35,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:35,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:35,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T10:16:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=203, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-12-09T10:16:35,810 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:35,811 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:35,821 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:35,822 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300a8 connected 2024-12-09T10:16:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-12-09T10:16:35,930 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300a8, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:35,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=204, ppid=203, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:36,104 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:36,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=204 2024-12-09T10:16:36,105 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-12-09T10:16:36,118 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:36,118 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:36,118 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:36,118 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be88859300a7, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:36,118 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be88859300a7, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:36,228 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=204}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3d007003 to 127.0.0.1:64103 2024-12-09T10:16:36,228 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=204}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:36,230 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be88859300a7, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:36,261 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:36,261 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:36,267 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=204 2024-12-09T10:16:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=204 2024-12-09T10:16:36,274 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:36,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=204, resume processing ppid=203 2024-12-09T10:16:36,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=204, ppid=203, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 324 msec 2024-12-09T10:16:36,294 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T10:16:36,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=203, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 503 msec 2024-12-09T10:16:36,318 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5bc9670a to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@15d5e13e 2024-12-09T10:16:36,339 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:36,339 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:36,340 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5a6e4f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:36,340 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:36,354 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:36,355 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be88859300aa connected 2024-12-09T10:16:36,359 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-12-09T10:16:36,420 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:36,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:36,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:36,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T10:16:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=205, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-12-09T10:16:36,477 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:36,478 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:36,491 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:36,492 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300ab connected 2024-12-09T10:16:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-12-09T10:16:36,615 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300ab, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:36,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=206, ppid=205, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-12-09T10:16:36,783 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:36,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=206 2024-12-09T10:16:36,784 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:36,790 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:36,790 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:36,790 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:36,790 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be88859300aa, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:36,791 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be88859300aa, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:36,910 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be88859300aa, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:36,910 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=206}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5bc9670a to 127.0.0.1:64103 2024-12-09T10:16:36,910 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=206}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:36,933 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:36,933 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:36,934 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=206 2024-12-09T10:16:36,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=206 2024-12-09T10:16:36,938 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:36,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=206, resume processing ppid=205 2024-12-09T10:16:36,940 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T10:16:36,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=206, ppid=205, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 315 msec 2024-12-09T10:16:36,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=205, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 513 msec 2024-12-09T10:16:36,984 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e47836f to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@552b0b1d 2024-12-09T10:16:37,011 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:37,011 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:37,012 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6de8a3ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:37,013 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:37,021 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:37,021 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:37,022 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be88859300ad connected 2024-12-09T10:16:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-12-09T10:16:37,059 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:37,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:37,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:37,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T10:16:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=207, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-12-09T10:16:37,099 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:37,100 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:37,119 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:37,130 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300ae connected 2024-12-09T10:16:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-12-09T10:16:37,238 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300ae, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:37,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=208, ppid=207, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-12-09T10:16:37,403 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:37,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=208 2024-12-09T10:16:37,407 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:37,420 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:37,420 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:37,420 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:37,420 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be88859300ad, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:37,420 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be88859300ad, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:37,535 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be88859300ad, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:37,536 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=208}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e47836f to 127.0.0.1:64103 2024-12-09T10:16:37,536 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=208}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:37,554 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:37,554 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:37,555 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=208 2024-12-09T10:16:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=208 2024-12-09T10:16:37,557 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:37,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=208, resume processing ppid=207 2024-12-09T10:16:37,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=208, ppid=207, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 310 msec 2024-12-09T10:16:37,561 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T10:16:37,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=207, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 498 msec 2024-12-09T10:16:37,596 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3610042f to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@29f61553 2024-12-09T10:16:37,627 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:37,628 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:37,628 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@75d157d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:37,629 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:37,638 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:37,639 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be88859300b0 connected 2024-12-09T10:16:37,643 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-12-09T10:16:37,692 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:37,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:37,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T10:16:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=209, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=209 2024-12-09T10:16:37,731 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:37,731 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:37,746 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:37,751 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300b1 connected 2024-12-09T10:16:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=209 2024-12-09T10:16:37,863 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300b1, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:37,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=210, ppid=209, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=209 2024-12-09T10:16:38,024 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:38,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=210 2024-12-09T10:16:38,025 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:38,028 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=210 2024-12-09T10:16:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=210 2024-12-09T10:16:38,037 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:38,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=210, resume processing ppid=209 2024-12-09T10:16:38,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=210, ppid=209, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 165 msec 2024-12-09T10:16:38,039 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T10:16:38,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=209, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 343 msec 2024-12-09T10:16:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=209 2024-12-09T10:16:38,330 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:38,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:38,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:38,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-09T10:16:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=211, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-12-09T10:16:38,365 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:38,366 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:38,430 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:38,431 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300b2 connected 2024-12-09T10:16:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-12-09T10:16:38,552 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300b2, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:38,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=212, ppid=211, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:38,644 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-12-09T10:16:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-12-09T10:16:38,739 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:38,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=212 2024-12-09T10:16:38,740 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:38,744 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:38,744 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:38,744 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:38,744 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be88859300b0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:38,745 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be88859300b0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:38,860 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be88859300b0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:38,860 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=212}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3610042f to 127.0.0.1:64103 2024-12-09T10:16:38,860 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=212}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:38,878 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:38,879 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:38,887 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=212 2024-12-09T10:16:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=212 2024-12-09T10:16:38,889 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:38,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=212, resume processing ppid=211 2024-12-09T10:16:38,892 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-09T10:16:38,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=212, ppid=211, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 303 msec 2024-12-09T10:16:38,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=211, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 555 msec 2024-12-09T10:16:38,924 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x456b98fd to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@353ce36c 2024-12-09T10:16:38,935 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:38,936 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:38,936 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@517cffce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:38,937 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:38,946 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:38,946 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be88859300b4 connected 2024-12-09T10:16:38,946 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-12-09T10:16:38,973 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:38,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:38,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-09T10:16:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=213, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-12-09T10:16:39,004 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:39,005 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:39,021 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:39,022 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300b5 connected 2024-12-09T10:16:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-12-09T10:16:39,129 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300b5, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:39,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=214, ppid=213, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:39,291 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:39,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=214 2024-12-09T10:16:39,292 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:39,293 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:39,293 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:39,293 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:39,293 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be88859300b4, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:39,293 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be88859300b4, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-12-09T10:16:39,401 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be88859300b4, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:39,402 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=214}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x456b98fd to 127.0.0.1:64103 2024-12-09T10:16:39,402 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=214}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:39,420 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:39,421 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:39,421 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=214 2024-12-09T10:16:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=214 2024-12-09T10:16:39,425 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:39,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=214, resume processing ppid=213 2024-12-09T10:16:39,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=214, ppid=213, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 286 msec 2024-12-09T10:16:39,428 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-09T10:16:39,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=213, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 448 msec 2024-12-09T10:16:39,461 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x20e0691e to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@557da411 2024-12-09T10:16:39,478 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:39,479 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:39,479 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@24728b9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:39,481 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:39,493 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:39,493 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:39,499 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be88859300b7 connected 2024-12-09T10:16:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-12-09T10:16:39,612 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:39,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:39,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=215, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=215 2024-12-09T10:16:39,623 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:39,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:39,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:39,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:39,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=216, ppid=215, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=215 2024-12-09T10:16:39,807 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:39,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=216 2024-12-09T10:16:39,809 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:39,810 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:39,810 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:39,810 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:39,810 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be88859300b7, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:39,810 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be88859300b7, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:39,929 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be88859300b7, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:39,930 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=216}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x20e0691e to 127.0.0.1:64103 2024-12-09T10:16:39,930 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=216}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:39,930 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=215 2024-12-09T10:16:39,944 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=216}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:39,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:39,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:39,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:39,981 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=216}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:39,982 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=216 2024-12-09T10:16:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=216 2024-12-09T10:16:39,993 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:40,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=216, resume processing ppid=215 2024-12-09T10:16:40,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=216, ppid=215, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 342 msec 2024-12-09T10:16:40,063 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:40,063 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:40,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=215, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 446 msec 2024-12-09T10:16:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=215 2024-12-09T10:16:40,250 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:40,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=217, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=217 2024-12-09T10:16:40,253 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:40,255 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=217, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:40,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=217, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-09T10:16:40,256 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=217 2024-12-09T10:16:40,360 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:40,361 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=218, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=218 2024-12-09T10:16:40,367 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:40,370 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=218, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:40,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=218, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=10 msec 2024-12-09T10:16:40,373 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=218 2024-12-09T10:16:40,481 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:40,514 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=214 (was 216), OpenFileDescriptor=442 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1369 (was 1375), ProcessCount=11 (was 11), AvailableMemoryMB=2509 (was 2893) 2024-12-09T10:16:40,540 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=214, OpenFileDescriptor=442, MaxFileDescriptor=1048576, SystemLoadAverage=1369, ProcessCount=11, AvailableMemoryMB=2507 2024-12-09T10:16:40,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=219, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-12-09T10:16:40,559 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:40,560 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:40,587 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:40,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-12-09T10:16:40,688 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:40,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=220, ppid=219, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-12-09T10:16:40,888 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:40,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=220 2024-12-09T10:16:40,889 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:40,906 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=220}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:40,928 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:40,991 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=220 2024-12-09T10:16:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=220 2024-12-09T10:16:40,998 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:41,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=220, resume processing ppid=219 2024-12-09T10:16:41,003 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:41,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=220, ppid=219, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec 2024-12-09T10:16:41,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=219, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 462 msec 2024-12-09T10:16:41,036 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x250c924c to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@46ac2a90 2024-12-09T10:16:41,094 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:41,094 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:41,095 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3f900df6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:41,096 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:41,124 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:41,130 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:41,138 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be88859300ba connected 2024-12-09T10:16:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-12-09T10:16:41,172 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:41,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:16:41,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3938): Client=jenkins//172.17.0.3 disable replication peer, id=1 2024-12-09T10:16:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=221, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-12-09T10:16:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=221 2024-12-09T10:16:41,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=222, ppid=221, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=221 2024-12-09T10:16:41,354 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:41,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=222 2024-12-09T10:16:41,354 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-12-09T10:16:41,355 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T10:16:41,355 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T10:16:41,355 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:41,355 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be88859300ba, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:41,356 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be88859300ba, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:41,463 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be88859300ba, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:41,463 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=222}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x250c924c to 127.0.0.1:64103 2024-12-09T10:16:41,463 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=222}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:41,483 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:41,483 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T10:16:41,483 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=222 2024-12-09T10:16:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=222 2024-12-09T10:16:41,485 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for DISABLE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:41,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=222, resume processing ppid=221 2024-12-09T10:16:41,489 INFO [PEWorker-1 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-12-09T10:16:41,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=222, ppid=221, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 289 msec 2024-12-09T10:16:41,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=221, state=SUCCESS; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 313 msec 2024-12-09T10:16:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=221 2024-12-09T10:16:41,499 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:16:41,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=223, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=223 2024-12-09T10:16:41,508 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:41,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:41,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:41,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:41,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=224, ppid=223, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:41,540 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6438c980 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@46be24c 2024-12-09T10:16:41,563 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:41,564 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:41,565 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@45e1a6ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:41,566 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:41,585 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:41,585 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be88859300bc connected 2024-12-09T10:16:41,587 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=223 2024-12-09T10:16:41,690 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:41,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=224 2024-12-09T10:16:41,691 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:41,691 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:41,691 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:41,691 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:41,691 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be88859300bc, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:41,691 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be88859300bc, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:41,808 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be88859300bc, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:41,808 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=224}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6438c980 to 127.0.0.1:64103 2024-12-09T10:16:41,808 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=224}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:41,809 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=223 2024-12-09T10:16:41,827 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=224}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:41,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:41,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:41,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:41,862 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=224}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:41,862 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=224 2024-12-09T10:16:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=224 2024-12-09T10:16:41,865 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:41,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=224, resume processing ppid=223 2024-12-09T10:16:41,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=224, ppid=223, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 342 msec 2024-12-09T10:16:41,882 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:41,882 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:41,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=223, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 382 msec 2024-12-09T10:16:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=223 2024-12-09T10:16:42,130 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:42,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=225, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=225 2024-12-09T10:16:42,136 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:42,138 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=225, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:42,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=225, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-09T10:16:42,139 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=225 2024-12-09T10:16:42,239 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:42,240 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=226, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=226 2024-12-09T10:16:42,242 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:42,242 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=226, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:42,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=226, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T10:16:42,243 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=226 2024-12-09T10:16:42,352 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:42,366 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=212 (was 214), OpenFileDescriptor=440 (was 442), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1369 (was 1369), ProcessCount=11 (was 11), AvailableMemoryMB=2468 (was 2507) 2024-12-09T10:16:42,379 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=212, OpenFileDescriptor=440, MaxFileDescriptor=1048576, SystemLoadAverage=1369, ProcessCount=11, AvailableMemoryMB=2472 2024-12-09T10:16:42,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=dummy_peer 2024-12-09T10:16:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=227, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=227 2024-12-09T10:16:42,382 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:42,383 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=227, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:42,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=227, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T10:16:42,384 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=227 2024-12-09T10:16:42,490 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-12-09T10:16:42,490 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=228, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:42,493 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=228 2024-12-09T10:16:42,494 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=228, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:42,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=228, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T10:16:42,495 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=228 2024-12-09T10:16:42,600 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:42,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=229, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=229 2024-12-09T10:16:42,602 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:42,603 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=229, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:42,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=229, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T10:16:42,604 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=229 2024-12-09T10:16:42,709 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:42,724 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=212 (was 212), OpenFileDescriptor=440 (was 440), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1363 (was 1369), ProcessCount=11 (was 11), AvailableMemoryMB=2461 (was 2472) 2024-12-09T10:16:42,739 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=212, OpenFileDescriptor=440, MaxFileDescriptor=1048576, SystemLoadAverage=1363, ProcessCount=11, AvailableMemoryMB=2460 2024-12-09T10:16:42,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=230, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-12-09T10:16:42,765 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:42,765 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:42,781 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:42,783 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300bd connected 2024-12-09T10:16:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-12-09T10:16:42,896 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300bd, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:42,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=231, ppid=230, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-12-09T10:16:43,071 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:43,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=231 2024-12-09T10:16:43,071 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:43,088 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=231}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:43,101 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:43,151 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=231 2024-12-09T10:16:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=231 2024-12-09T10:16:43,153 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:43,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=231, resume processing ppid=230 2024-12-09T10:16:43,154 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:43,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=231, ppid=230, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 234 msec 2024-12-09T10:16:43,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=230, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 414 msec 2024-12-09T10:16:43,179 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6a60c54e to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@513fbb75 2024-12-09T10:16:43,188 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:43,188 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:43,189 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7de2a300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:43,190 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:43,201 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:43,201 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:43,202 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be88859300bf connected 2024-12-09T10:16:43,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-12-09T10:16:43,380 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3971): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-09T10:16:43,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=232, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=232 2024-12-09T10:16:43,413 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=232 2024-12-09T10:16:43,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:43,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:43,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:43,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=233, ppid=232, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:43,682 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:43,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=233 2024-12-09T10:16:43,682 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:43,682 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:43,682 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:43,682 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:43,683 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be88859300bf, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:43,683 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be88859300bf, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=232 2024-12-09T10:16:43,828 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be88859300bf, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:43,828 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=233}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6a60c54e to 127.0.0.1:64103 2024-12-09T10:16:43,828 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=233}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:43,828 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:43,864 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=233}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:43,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:43,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:43,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:43,955 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=233}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:43,955 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=233 2024-12-09T10:16:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=233 2024-12-09T10:16:43,957 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:43,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=233, resume processing ppid=232 2024-12-09T10:16:43,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=233, ppid=232, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 427 msec 2024-12-09T10:16:43,985 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:43,985 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:43,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=232, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 601 msec 2024-12-09T10:16:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=232 2024-12-09T10:16:44,029 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:44,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=234, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=234 2024-12-09T10:16:44,031 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:44,031 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=234, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:44,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=234, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T10:16:44,033 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=234 2024-12-09T10:16:44,140 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:44,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=235, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:44,143 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=235 2024-12-09T10:16:44,144 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=235, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:44,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=235, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T10:16:44,145 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=235 2024-12-09T10:16:44,250 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:44,263 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=211 (was 212), OpenFileDescriptor=438 (was 440), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1363 (was 1363), ProcessCount=11 (was 11), AvailableMemoryMB=2397 (was 2460) 2024-12-09T10:16:44,273 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=211, OpenFileDescriptor=438, MaxFileDescriptor=1048576, SystemLoadAverage=1363, ProcessCount=11, AvailableMemoryMB=2396 2024-12-09T10:16:44,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-09T10:16:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] ipc.CallRunner(138): callId: 795 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.3:48524 deadline: 1733739464275, exception=java.io.IOException: Replication peer modification disabled 2024-12-09T10:16:44,276 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 1 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-12-09T10:16:44,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-09T10:16:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] ipc.CallRunner(138): callId: 796 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.3:48524 deadline: 1733739464379, exception=java.io.IOException: Replication peer modification disabled 2024-12-09T10:16:44,380 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 105 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-12-09T10:16:44,381 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-12-09T10:16:44.276Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-12-09T10:16:44.380Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-12-09T10:16:44,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=236, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=236 2024-12-09T10:16:44,390 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:44,390 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=236, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:44,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=236, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T10:16:44,391 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=236 2024-12-09T10:16:44,499 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:44,499 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=237, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=237 2024-12-09T10:16:44,501 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:44,501 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=237, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:44,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=237, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T10:16:44,502 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=237 2024-12-09T10:16:44,609 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:44,619 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=211 (was 211), OpenFileDescriptor=438 (was 438), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1363 (was 1363), ProcessCount=11 (was 11), AvailableMemoryMB=2394 (was 2396) 2024-12-09T10:16:44,627 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=211, OpenFileDescriptor=438, MaxFileDescriptor=1048576, SystemLoadAverage=1363, ProcessCount=11, AvailableMemoryMB=2394 2024-12-09T10:16:44,627 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=238, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-12-09T10:16:44,643 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:44,644 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:44,662 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:44,663 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300c0 connected 2024-12-09T10:16:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-12-09T10:16:44,821 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300c0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:44,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=239, ppid=238, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:44,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-12-09T10:16:44,998 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:44,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=239 2024-12-09T10:16:44,999 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:45,021 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=239}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:45,041 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:45,085 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=239 2024-12-09T10:16:45,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=239 2024-12-09T10:16:45,087 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:45,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=239, resume processing ppid=238 2024-12-09T10:16:45,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=239, ppid=238, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 240 msec 2024-12-09T10:16:45,089 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:45,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=238, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 462 msec 2024-12-09T10:16:45,118 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x65644c23 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7abbb825 2024-12-09T10:16:45,147 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:45,148 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:45,148 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1e966077, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:45,149 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:45,161 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:45,161 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:45,167 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be88859300c2 connected 2024-12-09T10:16:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-12-09T10:16:45,260 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:45,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:45,267 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=240, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=240 2024-12-09T10:16:45,289 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:45,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:45,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:45,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:45,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=241, ppid=240, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=240 2024-12-09T10:16:45,448 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:16:45,462 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:45,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=241 2024-12-09T10:16:45,463 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:45,463 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:45,463 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:45,463 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:45,463 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be88859300c2, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:45,463 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be88859300c2, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:45,583 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be88859300c2, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:45,583 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=241}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x65644c23 to 127.0.0.1:64103 2024-12-09T10:16:45,583 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=241}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:45,584 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=240 2024-12-09T10:16:45,601 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=241}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:45,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:45,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:45,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:45,618 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=241}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:45,618 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=241 2024-12-09T10:16:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=241 2024-12-09T10:16:45,623 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:45,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=241, resume processing ppid=240 2024-12-09T10:16:45,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=241, ppid=240, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 322 msec 2024-12-09T10:16:45,640 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:45,640 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:45,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=240, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 373 msec 2024-12-09T10:16:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=240 2024-12-09T10:16:45,900 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:45,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=242, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=242 2024-12-09T10:16:45,906 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:45,907 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=242, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:45,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=242, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-09T10:16:45,908 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=242 2024-12-09T10:16:46,020 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:46,039 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=211 (was 211), OpenFileDescriptor=438 (was 438), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1363 (was 1363), ProcessCount=11 (was 11), AvailableMemoryMB=2369 (was 2394) 2024-12-09T10:16:46,048 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=211, OpenFileDescriptor=438, MaxFileDescriptor=1048576, SystemLoadAverage=1363, ProcessCount=11, AvailableMemoryMB=2369 2024-12-09T10:16:46,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=243, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-12-09T10:16:46,070 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:46,071 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:46,088 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:46,088 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300c3 connected 2024-12-09T10:16:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-12-09T10:16:46,204 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300c3, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:46,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=244, ppid=243, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-12-09T10:16:46,401 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:46,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=244 2024-12-09T10:16:46,401 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:46,422 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=244}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:46,457 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:46,538 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=244 2024-12-09T10:16:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=244 2024-12-09T10:16:46,553 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:46,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=244, resume processing ppid=243 2024-12-09T10:16:46,555 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:46,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=244, ppid=243, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 310 msec 2024-12-09T10:16:46,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=243, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 507 msec 2024-12-09T10:16:46,597 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x74a67722 to 127.0.0.1:64103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@566de3 2024-12-09T10:16:46,622 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T10:16:46,622 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T10:16:46,622 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@145bb16d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:16:46,623 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:46,638 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:46,639 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x100be88859300c5 connected 2024-12-09T10:16:46,651 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T10:16:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-12-09T10:16:46,679 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:46,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:46,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3960): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-09T10:16:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=245, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T10:16:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=245 2024-12-09T10:16:46,702 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T10:16:46,703 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:64103 2024-12-09T10:16:46,722 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id0x0, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:16:46,735 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300c6 connected 2024-12-09T10:16:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=245 2024-12-09T10:16:46,875 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@44102a05check-peer-cluster-id-0x100be88859300c6, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:46,886 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=246, ppid=245, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:46,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=245 2024-12-09T10:16:47,038 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:47,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=246 2024-12-09T10:16:47,038 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T10:16:47,042 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=246 2024-12-09T10:16:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=246 2024-12-09T10:16:47,058 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:47,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=246, resume processing ppid=245 2024-12-09T10:16:47,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=246, ppid=245, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 173 msec 2024-12-09T10:16:47,070 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:64103:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-09T10:16:47,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=245, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 389 msec 2024-12-09T10:16:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=245 2024-12-09T10:16:47,309 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T10:16:47,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3948): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-09T10:16:47,311 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=247, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=247 2024-12-09T10:16:47,314 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:47,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:47,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:47,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:47,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=248, ppid=247, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=247 2024-12-09T10:16:47,473 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:47,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=248 2024-12-09T10:16:47,474 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:47,474 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:47,474 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:47,474 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T10:16:47,474 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x100be88859300c5, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:47,474 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x100be88859300c5, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:16:47,587 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=248}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x74a67722 to 127.0.0.1:64103 2024-12-09T10:16:47,587 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=248}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:47,588 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x100be88859300c5, quorum=127.0.0.1:64103, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:47,588 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:47,589 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=248}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:47,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:47,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:47,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:47,616 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=248}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:47,617 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=248 2024-12-09T10:16:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=248 2024-12-09T10:16:47,624 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:47,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=248, resume processing ppid=247 2024-12-09T10:16:47,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=248, ppid=247, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 302 msec 2024-12-09T10:16:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=247 2024-12-09T10:16:47,640 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:47,640 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:47,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=247, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 329 msec 2024-12-09T10:16:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=247 2024-12-09T10:16:47,941 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:47,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=249, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=249 2024-12-09T10:16:47,946 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:47,948 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=249, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:47,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=249, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-09T10:16:47,950 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=249 2024-12-09T10:16:48,050 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T10:16:48,051 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=250, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=250 2024-12-09T10:16:48,054 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:48,056 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=250, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:48,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=250, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-09T10:16:48,057 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=250 2024-12-09T10:16:48,164 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:48,197 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=211 (was 211), OpenFileDescriptor=438 (was 438), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1286 (was 1363), ProcessCount=11 (was 11), AvailableMemoryMB=2013 (was 2369) 2024-12-09T10:16:48,218 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=211, OpenFileDescriptor=438, MaxFileDescriptor=1048576, SystemLoadAverage=1286, ProcessCount=11, AvailableMemoryMB=2012 2024-12-09T10:16:48,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=251, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=251 2024-12-09T10:16:48,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=252, ppid=251, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T10:16:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=251 2024-12-09T10:16:48,415 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:48,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=252 2024-12-09T10:16:48,416 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=252}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T10:16:48,444 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=252}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T10:16:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=251 2024-12-09T10:16:48,564 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=252}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=252 2024-12-09T10:16:48,564 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(587): peerId=1, queueId=1 (queues=1) is replicating from cluster=0c6f32a7-e3f8-4a47-90f8-ff1b85bfb52d to cluster=0c6f32a7-e3f8-4a47-90f8-ff1b85bfb52d 2024-12-09T10:16:48,564 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSource(343): peerId=1, starting shipping worker for walGroupId=de59e52b2fff%2C37621%2C1733739327100 2024-12-09T10:16:48,564 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1 {}] regionserver.ReplicationSourceWALReader(112): peerClusterZnode=1, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-12-09T10:16:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=252 2024-12-09T10:16:48,565 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.shipperde59e52b2fff%2C37621%2C1733739327100,1 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: de59e52b2fff%2C37621%2C1733739327100 2024-12-09T10:16:48,565 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.wal-reader.de59e52b2fff%2C37621%2C1733739327100,1 {}] regionserver.WALEntryStream(260): Creating new reader hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/WALs/de59e52b2fff,37621,1733739327100/de59e52b2fff%2C37621%2C1733739327100.1733739331285, startPosition=0, beingWritten=true 2024-12-09T10:16:48,566 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:48,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=252, resume processing ppid=251 2024-12-09T10:16:48,568 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T10:16:48,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=252, ppid=251, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 303 msec 2024-12-09T10:16:48,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=251, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 348 msec 2024-12-09T10:16:48,586 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.wal-reader.de59e52b2fff%2C37621%2C1733739327100,1 {}] regionserver.ReplicationSourceWALReader(181): Read 0 WAL entries eligible for replication 2024-12-09T10:16:48,586 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.wal-reader.de59e52b2fff%2C37621%2C1733739327100,1 {}] regionserver.WALEntryStream(223): Reset reader hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/WALs/de59e52b2fff,37621,1733739327100/de59e52b2fff%2C37621%2C1733739327100.1733739331285 to pos 589, reset compression=false 2024-12-09T10:16:48,586 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.shipperde59e52b2fff%2C37621%2C1733739327100,1 {}] regionserver.ReplicationSourceShipper(110): Shipper from source 1 got entry batch from reader: WALEntryBatch [walEntries=[], lastWalPath=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/WALs/de59e52b2fff,37621,1733739327100/de59e52b2fff%2C37621%2C1733739327100.1733739331285, lastWalPosition=589, nbRowKeys=0, nbHFiles=0, heapSize=0, lastSeqIds={}, endOfFile=false,usedBufferSize=0] 2024-12-09T10:16:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=251 2024-12-09T10:16:48,850 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:48,850 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3919): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T10:16:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=253, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T10:16:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=253 2024-12-09T10:16:48,852 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-12-09T10:16:48,853 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=253, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:48,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=253, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec 2024-12-09T10:16:48,854 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=253 2024-12-09T10:16:48,960 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-12-09T10:16:48,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-09T10:16:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=254, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T10:16:48,963 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:49,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T10:16:49,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T10:16:49,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:49,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T10:16:49,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=255, ppid=254, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T10:16:49,231 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to de59e52b2fff,37621,1733739327100 2024-12-09T10:16:49,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37621 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=255 2024-12-09T10:16:49,232 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T10:16:49,232 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T10:16:49,232 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T10:16:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T10:16:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T10:16:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T10:16:50,232 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.wal-reader.de59e52b2fff%2C37621%2C1733739327100,1 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:131) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:166) ~[classes/:?] 2024-12-09T10:16:50,232 WARN [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.shipperde59e52b2fff%2C37621%2C1733739327100,1 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:313) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-12-09T10:16:51,085 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:16:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T10:16:51,232 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSource(711): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0.replicationSource,1.replicationSource.shipperde59e52b2fff%2C37621%2C1733739327100,1 terminated 2024-12-09T10:16:51,232 INFO [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T10:16:51,240 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=255}] zookeeper.ZKUtil(111): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:51,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1/de59e52b2fff%2C37621%2C1733739327100.1733739331285 2024-12-09T10:16:51,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:51,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/de59e52b2fff,37621,1733739327100/1 2024-12-09T10:16:51,276 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=255}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:51,276 DEBUG [RS_REFRESH_PEER-regionserver/de59e52b2fff:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=255 2024-12-09T10:16:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(4106): Remote procedure done, pid=255 2024-12-09T10:16:51,278 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on de59e52b2fff,37621,1733739327100 suceeded 2024-12-09T10:16:51,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=255, resume processing ppid=254 2024-12-09T10:16:51,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=255, ppid=254, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1980 sec 2024-12-09T10:16:51,293 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T10:16:51,294 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T10:16:51,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=254, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.3340 sec 2024-12-09T10:16:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T10:16:53,139 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T10:16:53,140 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.HMaster(3926): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-09T10:16:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] procedure2.ProcedureExecutor(1098): Stored pid=256, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T10:16:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=256 2024-12-09T10:16:53,141 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T10:16:53,142 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=256, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T10:16:53,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=256, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-09T10:16:53,142 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T10:16:53,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37695 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=256 2024-12-09T10:16:53,249 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T10:16:53,267 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=213 (was 211) - Thread LEAK? -, OpenFileDescriptor=443 (was 438) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1319 (was 1286) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1969 (was 2012) 2024-12-09T10:16:53,267 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-12-09T10:16:53,267 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:96) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T10:16:53,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:53,269 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x57d89e0d to 127.0.0.1:64103 2024-12-09T10:16:53,269 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-09T10:16:53,269 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7502c059 to 127.0.0.1:64103 2024-12-09T10:16:53,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:53,269 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T10:16:53,269 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1029449637, stopped=false 2024-12-09T10:16:53,270 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=de59e52b2fff,37695,1733739326017 2024-12-09T10:16:53,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T10:16:53,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T10:16:53,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:16:53,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:16:53,284 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-09T10:16:53,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:53,285 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T10:16:53,285 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T10:16:53,285 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'de59e52b2fff,37621,1733739327100' ***** 2024-12-09T10:16:53,285 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-09T10:16:53,286 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T10:16:53,286 INFO [RS:0;de59e52b2fff:37621 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T10:16:53,286 INFO [RS:0;de59e52b2fff:37621 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T10:16:53,286 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(3579): Received CLOSE for cb29027c56d4cda2e8d456ca713f729b 2024-12-09T10:16:53,286 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-09T10:16:53,295 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1224): stopping server de59e52b2fff,37621,1733739327100 2024-12-09T10:16:53,295 DEBUG [RS:0;de59e52b2fff:37621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:53,295 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T10:16:53,295 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T10:16:53,295 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T10:16:53,295 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-09T10:16:53,297 DEBUG [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing cb29027c56d4cda2e8d456ca713f729b, disabling compactions & flushes 2024-12-09T10:16:53,297 INFO [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. 2024-12-09T10:16:53,297 DEBUG [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. 2024-12-09T10:16:53,297 DEBUG [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. after waiting 0 ms 2024-12-09T10:16:53,297 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-12-09T10:16:53,297 DEBUG [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. 2024-12-09T10:16:53,297 DEBUG [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, cb29027c56d4cda2e8d456ca713f729b=hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b.} 2024-12-09T10:16:53,297 DEBUG [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-09T10:16:53,298 INFO [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-09T10:16:53,298 DEBUG [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-09T10:16:53,298 DEBUG [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T10:16:53,298 DEBUG [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T10:16:53,298 INFO [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing cb29027c56d4cda2e8d456ca713f729b 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-09T10:16:53,298 DEBUG [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, cb29027c56d4cda2e8d456ca713f729b 2024-12-09T10:16:53,298 INFO [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=1.23 KB heapSize=2.87 KB 2024-12-09T10:16:53,378 DEBUG [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740/.tmp/info/10859d978f084a4cbd13dbe35bbbcf61 is 143, key is hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b./info:regioninfo/1733739333626/Put/seqid=0 2024-12-09T10:16:53,378 DEBUG [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/namespace/cb29027c56d4cda2e8d456ca713f729b/.tmp/info/05811aedbd6242ec8444d97e654cdf46 is 45, key is default/info:d/1733739333761/Put/seqid=0 2024-12-09T10:16:53,382 INFO [regionserver/de59e52b2fff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T10:16:53,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741838_1014 (size=6595) 2024-12-09T10:16:53,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741837_1013 (size=5037) 2024-12-09T10:16:53,498 DEBUG [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, cb29027c56d4cda2e8d456ca713f729b 2024-12-09T10:16:53,699 DEBUG [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, cb29027c56d4cda2e8d456ca713f729b 2024-12-09T10:16:53,799 INFO [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/namespace/cb29027c56d4cda2e8d456ca713f729b/.tmp/info/05811aedbd6242ec8444d97e654cdf46 2024-12-09T10:16:53,799 INFO [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.14 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740/.tmp/info/10859d978f084a4cbd13dbe35bbbcf61 2024-12-09T10:16:53,859 DEBUG [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/namespace/cb29027c56d4cda2e8d456ca713f729b/.tmp/info/05811aedbd6242ec8444d97e654cdf46 as hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/namespace/cb29027c56d4cda2e8d456ca713f729b/info/05811aedbd6242ec8444d97e654cdf46 2024-12-09T10:16:53,876 INFO [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/namespace/cb29027c56d4cda2e8d456ca713f729b/info/05811aedbd6242ec8444d97e654cdf46, entries=2, sequenceid=6, filesize=4.9 K 2024-12-09T10:16:53,878 DEBUG [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740/.tmp/table/6dd6ac7be3ed4df7a2c60568d6de0c01 is 51, key is hbase:namespace/table:state/1733739333651/Put/seqid=0 2024-12-09T10:16:53,879 INFO [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for cb29027c56d4cda2e8d456ca713f729b in 581ms, sequenceid=6, compaction requested=false 2024-12-09T10:16:53,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741839_1015 (size=5242) 2024-12-09T10:16:53,887 INFO [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740/.tmp/table/6dd6ac7be3ed4df7a2c60568d6de0c01 2024-12-09T10:16:53,898 DEBUG [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/namespace/cb29027c56d4cda2e8d456ca713f729b/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-09T10:16:53,899 DEBUG [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, cb29027c56d4cda2e8d456ca713f729b 2024-12-09T10:16:53,901 INFO [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. 2024-12-09T10:16:53,901 DEBUG [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for cb29027c56d4cda2e8d456ca713f729b: 2024-12-09T10:16:53,901 DEBUG [RS_CLOSE_REGION-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733739332857.cb29027c56d4cda2e8d456ca713f729b. 2024-12-09T10:16:53,909 DEBUG [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740/.tmp/info/10859d978f084a4cbd13dbe35bbbcf61 as hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740/info/10859d978f084a4cbd13dbe35bbbcf61 2024-12-09T10:16:53,928 INFO [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740/info/10859d978f084a4cbd13dbe35bbbcf61, entries=10, sequenceid=9, filesize=6.4 K 2024-12-09T10:16:53,931 DEBUG [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740/.tmp/table/6dd6ac7be3ed4df7a2c60568d6de0c01 as hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740/table/6dd6ac7be3ed4df7a2c60568d6de0c01 2024-12-09T10:16:53,954 INFO [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740/table/6dd6ac7be3ed4df7a2c60568d6de0c01, entries=2, sequenceid=9, filesize=5.1 K 2024-12-09T10:16:53,957 INFO [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~1.23 KB/1264, heapSize ~2.59 KB/2648, currentSize=0 B/0 for 1588230740 in 659ms, sequenceid=9, compaction requested=false 2024-12-09T10:16:54,023 DEBUG [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-09T10:16:54,025 DEBUG [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T10:16:54,026 INFO [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-09T10:16:54,026 DEBUG [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-09T10:16:54,026 DEBUG [RS_CLOSE_META-regionserver/de59e52b2fff:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T10:16:54,099 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1250): stopping server de59e52b2fff,37621,1733739327100; all regions closed. 2024-12-09T10:16:54,103 INFO [regionserver/de59e52b2fff:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T10:16:54,103 INFO [regionserver/de59e52b2fff:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T10:16:54,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741834_1010 (size=2484) 2024-12-09T10:16:54,124 DEBUG [RS:0;de59e52b2fff:37621 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/oldWALs 2024-12-09T10:16:54,124 INFO [RS:0;de59e52b2fff:37621 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL de59e52b2fff%2C37621%2C1733739327100.meta:.meta(num 1733739332119) 2024-12-09T10:16:54,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741833_1009 (size=1414) 2024-12-09T10:16:54,146 DEBUG [RS:0;de59e52b2fff:37621 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/oldWALs 2024-12-09T10:16:54,146 INFO [RS:0;de59e52b2fff:37621 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL de59e52b2fff%2C37621%2C1733739327100:(num 1733739331285) 2024-12-09T10:16:54,146 DEBUG [RS:0;de59e52b2fff:37621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:54,146 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T10:16:54,147 INFO [RS:0;de59e52b2fff:37621 {}] hbase.ChoreService(370): Chore service for: regionserver/de59e52b2fff:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-09T10:16:54,147 INFO [regionserver/de59e52b2fff:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-09T10:16:54,148 INFO [RS:0;de59e52b2fff:37621 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.3:37621 2024-12-09T10:16:54,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/de59e52b2fff,37621,1733739327100 2024-12-09T10:16:54,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T10:16:54,331 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [de59e52b2fff,37621,1733739327100] 2024-12-09T10:16:54,331 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing de59e52b2fff,37621,1733739327100; numProcessing=1 2024-12-09T10:16:54,583 INFO [RS:0;de59e52b2fff:37621 {}] regionserver.HRegionServer(1307): Exiting; stopping=de59e52b2fff,37621,1733739327100; zookeeper connection closed. 2024-12-09T10:16:54,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:54,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37621-0x100be8885930001, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:54,583 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@55dc0b7d {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@55dc0b7d 2024-12-09T10:16:54,584 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-09T10:16:54,627 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/de59e52b2fff,37621,1733739327100 already deleted, retry=false 2024-12-09T10:16:54,627 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; de59e52b2fff,37621,1733739327100 expired; onlineServers=0 2024-12-09T10:16:54,627 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'de59e52b2fff,37695,1733739326017' ***** 2024-12-09T10:16:54,627 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T10:16:54,628 DEBUG [M:0;de59e52b2fff:37695 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f8babc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=de59e52b2fff/172.17.0.3:0 2024-12-09T10:16:54,628 INFO [M:0;de59e52b2fff:37695 {}] regionserver.HRegionServer(1224): stopping server de59e52b2fff,37695,1733739326017 2024-12-09T10:16:54,628 INFO [M:0;de59e52b2fff:37695 {}] regionserver.HRegionServer(1250): stopping server de59e52b2fff,37695,1733739326017; all regions closed. 2024-12-09T10:16:54,628 DEBUG [M:0;de59e52b2fff:37695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:16:54,628 DEBUG [M:0;de59e52b2fff:37695 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T10:16:54,628 DEBUG [M:0;de59e52b2fff:37695 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T10:16:54,628 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T10:16:54,628 DEBUG [master/de59e52b2fff:0:becomeActiveMaster-HFileCleaner.large.0-1733739330681 {}] cleaner.HFileCleaner(306): Exit Thread[master/de59e52b2fff:0:becomeActiveMaster-HFileCleaner.large.0-1733739330681,5,FailOnTimeoutGroup] 2024-12-09T10:16:54,628 DEBUG [master/de59e52b2fff:0:becomeActiveMaster-HFileCleaner.small.0-1733739330685 {}] cleaner.HFileCleaner(306): Exit Thread[master/de59e52b2fff:0:becomeActiveMaster-HFileCleaner.small.0-1733739330685,5,FailOnTimeoutGroup] 2024-12-09T10:16:54,628 INFO [M:0;de59e52b2fff:37695 {}] hbase.ChoreService(370): Chore service for: master/de59e52b2fff:0 had [] on shutdown 2024-12-09T10:16:54,628 DEBUG [M:0;de59e52b2fff:37695 {}] master.HMaster(1733): Stopping service threads 2024-12-09T10:16:54,628 INFO [M:0;de59e52b2fff:37695 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T10:16:54,629 INFO [M:0;de59e52b2fff:37695 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T10:16:54,629 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T10:16:54,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T10:16:54,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:16:54,805 DEBUG [M:0;de59e52b2fff:37695 {}] zookeeper.ZKUtil(347): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T10:16:54,805 WARN [M:0;de59e52b2fff:37695 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T10:16:54,805 INFO [M:0;de59e52b2fff:37695 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-09T10:16:54,805 INFO [M:0;de59e52b2fff:37695 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T10:16:54,805 DEBUG [M:0;de59e52b2fff:37695 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T10:16:54,806 INFO [M:0;de59e52b2fff:37695 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:16:54,806 DEBUG [M:0;de59e52b2fff:37695 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:16:54,806 DEBUG [M:0;de59e52b2fff:37695 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T10:16:54,806 DEBUG [M:0;de59e52b2fff:37695 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:16:54,806 INFO [M:0;de59e52b2fff:37695 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=745.75 KB heapSize=893.27 KB 2024-12-09T10:16:54,808 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T10:16:54,876 DEBUG [M:0;de59e52b2fff:37695 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a62964451dc545c3bdd148012b89d5ce is 82, key is hbase:meta,,1/info:regioninfo/1733739332522/Put/seqid=0 2024-12-09T10:16:54,922 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T10:16:54,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741840_1016 (size=5672) 2024-12-09T10:16:54,950 INFO [M:0;de59e52b2fff:37695 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1791 (bloomFilter=true), to=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a62964451dc545c3bdd148012b89d5ce 2024-12-09T10:16:55,130 DEBUG [M:0;de59e52b2fff:37695 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/96faaab8c0c841e5a6c88116b8e459ed is 2089, key is \x00\x00\x00\x00\x00\x00\x00\x9D/proc:d/1733739383920/Put/seqid=0 2024-12-09T10:16:55,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741841_1017 (size=167152) 2024-12-09T10:16:55,565 INFO [M:0;de59e52b2fff:37695 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=743.19 KB at sequenceid=1791 (bloomFilter=true), to=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/96faaab8c0c841e5a6c88116b8e459ed 2024-12-09T10:16:55,601 INFO [M:0;de59e52b2fff:37695 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 96faaab8c0c841e5a6c88116b8e459ed 2024-12-09T10:16:55,653 DEBUG [M:0;de59e52b2fff:37695 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/017c5f4f9af14eae9e4037fa142bb30e is 69, key is de59e52b2fff,37621,1733739327100/rs:state/1733739330822/Put/seqid=0 2024-12-09T10:16:55,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741842_1018 (size=5156) 2024-12-09T10:16:55,695 INFO [M:0;de59e52b2fff:37695 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1791 (bloomFilter=true), to=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/017c5f4f9af14eae9e4037fa142bb30e 2024-12-09T10:16:55,742 DEBUG [M:0;de59e52b2fff:37695 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f333e385694c48d3a59b12ea6c811135 is 68, key is replication_peer_modification_on/state:d/1733739413250/Put/seqid=0 2024-12-09T10:16:55,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741843_1019 (size=5154) 2024-12-09T10:16:56,148 INFO [M:0;de59e52b2fff:37695 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2 KB at sequenceid=1791 (bloomFilter=true), to=hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f333e385694c48d3a59b12ea6c811135 2024-12-09T10:16:56,157 DEBUG [M:0;de59e52b2fff:37695 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a62964451dc545c3bdd148012b89d5ce as hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a62964451dc545c3bdd148012b89d5ce 2024-12-09T10:16:56,165 INFO [M:0;de59e52b2fff:37695 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a62964451dc545c3bdd148012b89d5ce, entries=8, sequenceid=1791, filesize=5.5 K 2024-12-09T10:16:56,167 DEBUG [M:0;de59e52b2fff:37695 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/96faaab8c0c841e5a6c88116b8e459ed as hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/96faaab8c0c841e5a6c88116b8e459ed 2024-12-09T10:16:56,174 INFO [M:0;de59e52b2fff:37695 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 96faaab8c0c841e5a6c88116b8e459ed 2024-12-09T10:16:56,175 INFO [M:0;de59e52b2fff:37695 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/96faaab8c0c841e5a6c88116b8e459ed, entries=256, sequenceid=1791, filesize=163.2 K 2024-12-09T10:16:56,176 DEBUG [M:0;de59e52b2fff:37695 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/017c5f4f9af14eae9e4037fa142bb30e as hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/017c5f4f9af14eae9e4037fa142bb30e 2024-12-09T10:16:56,184 INFO [M:0;de59e52b2fff:37695 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/017c5f4f9af14eae9e4037fa142bb30e, entries=1, sequenceid=1791, filesize=5.0 K 2024-12-09T10:16:56,186 DEBUG [M:0;de59e52b2fff:37695 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f333e385694c48d3a59b12ea6c811135 as hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f333e385694c48d3a59b12ea6c811135 2024-12-09T10:16:56,194 INFO [M:0;de59e52b2fff:37695 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33961/user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f333e385694c48d3a59b12ea6c811135, entries=1, sequenceid=1791, filesize=5.0 K 2024-12-09T10:16:56,195 INFO [M:0;de59e52b2fff:37695 {}] regionserver.HRegion(3040): Finished flush of dataSize ~745.75 KB/763643, heapSize ~893.20 KB/914640, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1389ms, sequenceid=1791, compaction requested=false 2024-12-09T10:16:56,200 INFO [M:0;de59e52b2fff:37695 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:16:56,200 DEBUG [M:0;de59e52b2fff:37695 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-09T10:16:56,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34711 is added to blk_1073741830_1006 (size=881283) 2024-12-09T10:16:56,211 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(743): complete file /user/jenkins/test-data/cb86d3d8-75c0-1f79-5715-d2d52a4718fc/MasterData/WALs/de59e52b2fff,37695,1733739326017/de59e52b2fff%2C37695%2C1733739326017.1733739329364 not finished, retry = 0 2024-12-09T10:16:56,313 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-09T10:16:56,313 INFO [M:0;de59e52b2fff:37695 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-09T10:16:56,313 INFO [M:0;de59e52b2fff:37695 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.3:37695 2024-12-09T10:16:56,329 DEBUG [M:0;de59e52b2fff:37695 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/de59e52b2fff,37695,1733739326017 already deleted, retry=false 2024-12-09T10:16:56,461 INFO [M:0;de59e52b2fff:37695 {}] regionserver.HRegionServer(1307): Exiting; stopping=de59e52b2fff,37695,1733739326017; zookeeper connection closed. 2024-12-09T10:16:56,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:56,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37695-0x100be8885930000, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:56,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100be8885930019, quorum=127.0.0.1:64103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:16:56,606 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7d75efc4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T10:16:56,609 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@723ec49{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T10:16:56,609 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T10:16:56,610 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a2745b0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T10:16:56,610 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5854846b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/hadoop.log.dir/,STOPPED} 2024-12-09T10:16:56,613 WARN [BP-1908975505-172.17.0.3-1733739321312 heartbeating to localhost/127.0.0.1:33961 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T10:16:56,613 WARN [BP-1908975505-172.17.0.3-1733739321312 heartbeating to localhost/127.0.0.1:33961 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1908975505-172.17.0.3-1733739321312 (Datanode Uuid 197ca379-f6d4-4524-baa8-d926b31d84e5) service to localhost/127.0.0.1:33961 2024-12-09T10:16:56,615 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/cluster_4c6bf60c-533f-d07c-4add-3b892b5aceba/dfs/data/data1/current/BP-1908975505-172.17.0.3-1733739321312 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T10:16:56,615 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T10:16:56,615 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/cluster_4c6bf60c-533f-d07c-4add-3b892b5aceba/dfs/data/data2/current/BP-1908975505-172.17.0.3-1733739321312 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T10:16:56,615 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T10:16:56,615 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T10:16:56,624 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2490b214{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T10:16:56,624 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1f7e2804{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T10:16:56,624 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T10:16:56,625 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@201ce359{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T10:16:56,625 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@842d13f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/33e1bf47-1cc4-42de-855c-ae6099450540/hadoop.log.dir/,STOPPED} 2024-12-09T10:16:56,637 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-09T10:16:56,654 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down