2024-11-09 02:20:29,460 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f
2024-11-09 02:20:29,474 main DEBUG Took 0.011568 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-09 02:20:29,474 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-09 02:20:29,475 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-09 02:20:29,476 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-09 02:20:29,484 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 02:20:29,494 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-09 02:20:29,508 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,509 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 02:20:29,510 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,510 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 02:20:29,511 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,511 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 02:20:29,512 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,512 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 02:20:29,513 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,513 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 02:20:29,514 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,514 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 02:20:29,515 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,515 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 02:20:29,515 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,515 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 02:20:29,516 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,516 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 02:20:29,517 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,517 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 02:20:29,518 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,518 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 02:20:29,518 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,518 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 02:20:29,519 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,519 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-09 02:20:29,521 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 02:20:29,522 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-09 02:20:29,524 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-09 02:20:29,524 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-09 02:20:29,526 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-09 02:20:29,526 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-09 02:20:29,535 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-09 02:20:29,537 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-09 02:20:29,539 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-09 02:20:29,540 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-09 02:20:29,540 main DEBUG createAppenders(={Console})
2024-11-09 02:20:29,541 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f initialized
2024-11-09 02:20:29,541 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f
2024-11-09 02:20:29,541 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f OK.
2024-11-09 02:20:29,542 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-09 02:20:29,542 main DEBUG OutputStream closed
2024-11-09 02:20:29,542 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-09 02:20:29,543 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-09 02:20:29,543 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@505fc5a4 OK
2024-11-09 02:20:29,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-09 02:20:29,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-09 02:20:29,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-09 02:20:29,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-09 02:20:29,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-09 02:20:29,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-09 02:20:29,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-09 02:20:29,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-09 02:20:29,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-09 02:20:29,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-09 02:20:29,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-09 02:20:29,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-09 02:20:29,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-09 02:20:29,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-09 02:20:29,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-09 02:20:29,623 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-09 02:20:29,623 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-09 02:20:29,624 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-09 02:20:29,626 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-09 02:20:29,626 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@39de3d36) with optional ClassLoader: null
2024-11-09 02:20:29,627 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-09 02:20:29,627 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@39de3d36] started OK.
2024-11-09T02:20:29,877 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6
2024-11-09 02:20:29,880 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-09 02:20:29,881 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-09T02:20:29,907 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi timeout: 26 mins
2024-11-09T02:20:29,926 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-09T02:20:29,930 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/cluster_b0eb3e89-6ba9-0dea-c5b3-6b36556d49a8, deleteOnExit=true
2024-11-09T02:20:29,931 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-11-09T02:20:29,932 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/test.cache.data in system properties and HBase conf
2024-11-09T02:20:29,933 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/hadoop.tmp.dir in system properties and HBase conf
2024-11-09T02:20:29,933 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/hadoop.log.dir in system properties and HBase conf
2024-11-09T02:20:29,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-09T02:20:29,935 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-09T02:20:29,935 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-11-09T02:20:30,040 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-09T02:20:30,131 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-09T02:20:30,135 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-09T02:20:30,136 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-09T02:20:30,136 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-09T02:20:30,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T02:20:30,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-09T02:20:30,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-09T02:20:30,138 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T02:20:30,138 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T02:20:30,139 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-09T02:20:30,139 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/nfs.dump.dir in system properties and HBase conf
2024-11-09T02:20:30,139 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/java.io.tmpdir in system properties and HBase conf
2024-11-09T02:20:30,140 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T02:20:30,140 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-09T02:20:30,141 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-09T02:20:30,969 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-09T02:20:31,069 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2291ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-09T02:20:31,168 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T02:20:31,237 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T02:20:31,258 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T02:20:31,259 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T02:20:31,260 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T02:20:31,274 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T02:20:31,277 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@842d13f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/hadoop.log.dir/,AVAILABLE}
2024-11-09T02:20:31,278 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@201ce359{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T02:20:31,478 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2490b214{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/java.io.tmpdir/jetty-localhost-45143-hadoop-hdfs-3_4_1-tests_jar-_-any-13707619271278605438/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T02:20:31,486 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1f7e2804{HTTP/1.1, (http/1.1)}{localhost:45143}
2024-11-09T02:20:31,487 INFO  [Time-limited test {}] server.Server(415): Started @2709ms
2024-11-09T02:20:31,898 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T02:20:31,905 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T02:20:31,907 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T02:20:31,908 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T02:20:31,908 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T02:20:31,909 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5854846b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/hadoop.log.dir/,AVAILABLE}
2024-11-09T02:20:31,910 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a2745b0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T02:20:32,032 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7d75efc4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/java.io.tmpdir/jetty-localhost-43405-hadoop-hdfs-3_4_1-tests_jar-_-any-6330617240492941946/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T02:20:32,033 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@723ec49{HTTP/1.1, (http/1.1)}{localhost:43405}
2024-11-09T02:20:32,034 INFO  [Time-limited test {}] server.Server(415): Started @3256ms
2024-11-09T02:20:32,093 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T02:20:32,590 WARN  [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/cluster_b0eb3e89-6ba9-0dea-c5b3-6b36556d49a8/dfs/data/data1/current/BP-956810084-172.17.0.2-1731118830705/current, will proceed with Du for space computation calculation, 
2024-11-09T02:20:32,590 WARN  [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/cluster_b0eb3e89-6ba9-0dea-c5b3-6b36556d49a8/dfs/data/data2/current/BP-956810084-172.17.0.2-1731118830705/current, will proceed with Du for space computation calculation, 
2024-11-09T02:20:32,621 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T02:20:32,677 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6
2024-11-09T02:20:32,681 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x29781ca3a603cde4 with lease ID 0xa17be8f8909e3a0c: Processing first storage report for DS-e2fd8557-6f9d-43d5-826f-6bbf1bb53ac1 from datanode DatanodeRegistration(127.0.0.1:34107, datanodeUuid=bef3e937-7983-4496-be6f-0e357e7c36ea, infoPort=46621, infoSecurePort=0, ipcPort=46271, storageInfo=lv=-57;cid=testClusterID;nsid=674384793;c=1731118830705)
2024-11-09T02:20:32,683 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x29781ca3a603cde4 with lease ID 0xa17be8f8909e3a0c: from storage DS-e2fd8557-6f9d-43d5-826f-6bbf1bb53ac1 node DatanodeRegistration(127.0.0.1:34107, datanodeUuid=bef3e937-7983-4496-be6f-0e357e7c36ea, infoPort=46621, infoSecurePort=0, ipcPort=46271, storageInfo=lv=-57;cid=testClusterID;nsid=674384793;c=1731118830705), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-11-09T02:20:32,684 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x29781ca3a603cde4 with lease ID 0xa17be8f8909e3a0c: Processing first storage report for DS-6c967a77-3b55-41cc-9dc9-f5d314268fe4 from datanode DatanodeRegistration(127.0.0.1:34107, datanodeUuid=bef3e937-7983-4496-be6f-0e357e7c36ea, infoPort=46621, infoSecurePort=0, ipcPort=46271, storageInfo=lv=-57;cid=testClusterID;nsid=674384793;c=1731118830705)
2024-11-09T02:20:32,684 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x29781ca3a603cde4 with lease ID 0xa17be8f8909e3a0c: from storage DS-6c967a77-3b55-41cc-9dc9-f5d314268fe4 node DatanodeRegistration(127.0.0.1:34107, datanodeUuid=bef3e937-7983-4496-be6f-0e357e7c36ea, infoPort=46621, infoSecurePort=0, ipcPort=46271, storageInfo=lv=-57;cid=testClusterID;nsid=674384793;c=1731118830705), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-09T02:20:32,763 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/cluster_b0eb3e89-6ba9-0dea-c5b3-6b36556d49a8/zookeeper_0, clientPort=60825, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/cluster_b0eb3e89-6ba9-0dea-c5b3-6b36556d49a8/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/cluster_b0eb3e89-6ba9-0dea-c5b3-6b36556d49a8/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-09T02:20:32,776 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=60825
2024-11-09T02:20:32,786 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T02:20:32,789 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T02:20:33,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741825_1001 (size=7)
2024-11-09T02:20:33,438 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91 with version=8
2024-11-09T02:20:33,438 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/hbase-staging
2024-11-09T02:20:33,568 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-09T02:20:33,846 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/54f3333e9615:0 server-side Connection retries=6
2024-11-09T02:20:33,867 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T02:20:33,868 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T02:20:33,868 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T02:20:33,868 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T02:20:33,869 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T02:20:34,006 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-09T02:20:34,067 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-09T02:20:34,077 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-09T02:20:34,081 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T02:20:34,109 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 17373 (auto-detected)
2024-11-09T02:20:34,110 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-09T02:20:34,129 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:40627
2024-11-09T02:20:34,138 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T02:20:34,140 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T02:20:34,152 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:40627 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:34,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:406270x0, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:34,188 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40627-0x1018ad491d00000 connected
2024-11-09T02:20:34,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-09T02:20:34,219 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T02:20:34,222 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T02:20:34,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40627
2024-11-09T02:20:34,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40627
2024-11-09T02:20:34,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40627
2024-11-09T02:20:34,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40627
2024-11-09T02:20:34,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40627
2024-11-09T02:20:34,235 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91, hbase.cluster.distributed=false
2024-11-09T02:20:34,295 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/54f3333e9615:0 server-side Connection retries=6
2024-11-09T02:20:34,296 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T02:20:34,296 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T02:20:34,296 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T02:20:34,296 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T02:20:34,296 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T02:20:34,299 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-09T02:20:34,301 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T02:20:34,302 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:37407
2024-11-09T02:20:34,303 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-09T02:20:34,308 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-09T02:20:34,310 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T02:20:34,312 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T02:20:34,316 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:37407 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:34,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374070x0, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:34,320 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37407-0x1018ad491d00001 connected
2024-11-09T02:20:34,320 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-09T02:20:34,322 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T02:20:34,323 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T02:20:34,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37407
2024-11-09T02:20:34,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37407
2024-11-09T02:20:34,327 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37407
2024-11-09T02:20:34,327 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37407
2024-11-09T02:20:34,328 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37407
2024-11-09T02:20:34,330 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/54f3333e9615,40627,1731118833561
2024-11-09T02:20:34,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T02:20:34,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T02:20:34,339 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/54f3333e9615,40627,1731118833561
2024-11-09T02:20:34,346 DEBUG [M:0;54f3333e9615:40627 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;54f3333e9615:40627
2024-11-09T02:20:34,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-09T02:20:34,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-09T02:20:34,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:34,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:34,360 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-09T02:20:34,361 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/54f3333e9615,40627,1731118833561 from backup master directory
2024-11-09T02:20:34,361 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-09T02:20:34,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/54f3333e9615,40627,1731118833561
2024-11-09T02:20:34,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T02:20:34,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T02:20:34,364 WARN  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T02:20:34,364 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=54f3333e9615,40627,1731118833561
2024-11-09T02:20:34,366 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-09T02:20:34,368 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-09T02:20:34,432 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741826_1002 (size=42)
2024-11-09T02:20:34,836 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/hbase.id with ID: 2c4c9e0c-e2c2-40e8-aa88-88b5f5265614
2024-11-09T02:20:34,877 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T02:20:34,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:34,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:34,917 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741827_1003 (size=196)
2024-11-09T02:20:35,336 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-09T02:20:35,340 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-09T02:20:35,357 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396
java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo)
	at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.<clinit>(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:232) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:207) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:35,362 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-09T02:20:35,391 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741828_1004 (size=1189)
2024-11-09T02:20:35,811 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store
2024-11-09T02:20:35,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741829_1005 (size=34)
2024-11-09T02:20:36,232 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-09T02:20:36,233 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T02:20:36,234 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T02:20:36,234 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T02:20:36,234 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T02:20:36,234 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T02:20:36,234 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T02:20:36,234 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T02:20:36,234 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-11-09T02:20:36,237 WARN  [master/54f3333e9615:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/.initializing
2024-11-09T02:20:36,237 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/WALs/54f3333e9615,40627,1731118833561
2024-11-09T02:20:36,243 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-11-09T02:20:36,254 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54f3333e9615%2C40627%2C1731118833561, suffix=, logDir=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/WALs/54f3333e9615,40627,1731118833561, archiveDir=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/oldWALs, maxLogs=10
2024-11-09T02:20:36,275 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/WALs/54f3333e9615,40627,1731118833561/54f3333e9615%2C40627%2C1731118833561.1731118836258, exclude list is [], retry=0
2024-11-09T02:20:36,292 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34107,DS-e2fd8557-6f9d-43d5-826f-6bbf1bb53ac1,DISK]
2024-11-09T02:20:36,296 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf.
2024-11-09T02:20:36,334 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/WALs/54f3333e9615,40627,1731118833561/54f3333e9615%2C40627%2C1731118833561.1731118836258
2024-11-09T02:20:36,335 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46621:46621)]
2024-11-09T02:20:36,335 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-09T02:20:36,336 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T02:20:36,339 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T02:20:36,340 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T02:20:36,380 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T02:20:36,404 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-09T02:20:36,408 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T02:20:36,411 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T02:20:36,412 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T02:20:36,415 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-09T02:20:36,415 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T02:20:36,416 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T02:20:36,417 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T02:20:36,419 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-09T02:20:36,420 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T02:20:36,421 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T02:20:36,421 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T02:20:36,424 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-09T02:20:36,424 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T02:20:36,425 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T02:20:36,429 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T02:20:36,430 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T02:20:36,439 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-09T02:20:36,442 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T02:20:36,447 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T02:20:36,448 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70392775, jitterRate=0.04893408715724945}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-09T02:20:36,452 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-11-09T02:20:36,453 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-09T02:20:36,484 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@189db86e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:36,520 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-11-09T02:20:36,532 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-09T02:20:36,533 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-09T02:20:36,535 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-09T02:20:36,537 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec
2024-11-09T02:20:36,543 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 5 msec
2024-11-09T02:20:36,543 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-09T02:20:36,570 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-09T02:20:36,583 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-09T02:20:36,585 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-11-09T02:20:36,588 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-09T02:20:36,589 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-09T02:20:36,591 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-11-09T02:20:36,593 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-09T02:20:36,596 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-09T02:20:36,598 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-11-09T02:20:36,599 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-09T02:20:36,601 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-11-09T02:20:36,610 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-09T02:20:36,612 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-09T02:20:36,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T02:20:36,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T02:20:36,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:36,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:36,617 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=54f3333e9615,40627,1731118833561, sessionid=0x1018ad491d00000, setting cluster-up flag (Was=false)
2024-11-09T02:20:36,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:36,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:36,635 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-09T02:20:36,636 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=54f3333e9615,40627,1731118833561
2024-11-09T02:20:36,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:36,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:36,647 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-09T02:20:36,648 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=54f3333e9615,40627,1731118833561
2024-11-09T02:20:36,728 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-11-09T02:20:36,734 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-11-09T02:20:36,737 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-09T02:20:36,742 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 54f3333e9615,40627,1731118833561
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-09T02:20:36,745 DEBUG [RS:0;54f3333e9615:37407 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;54f3333e9615:37407
2024-11-09T02:20:36,746 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/54f3333e9615:0, corePoolSize=5, maxPoolSize=5
2024-11-09T02:20:36,746 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/54f3333e9615:0, corePoolSize=5, maxPoolSize=5
2024-11-09T02:20:36,746 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/54f3333e9615:0, corePoolSize=5, maxPoolSize=5
2024-11-09T02:20:36,747 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/54f3333e9615:0, corePoolSize=5, maxPoolSize=5
2024-11-09T02:20:36,747 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/54f3333e9615:0, corePoolSize=10, maxPoolSize=10
2024-11-09T02:20:36,747 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/54f3333e9615:0, corePoolSize=1, maxPoolSize=1
2024-11-09T02:20:36,747 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/54f3333e9615:0, corePoolSize=2, maxPoolSize=2
2024-11-09T02:20:36,747 INFO  [RS:0;54f3333e9615:37407 {}] regionserver.HRegionServer(1008): ClusterId : 2c4c9e0c-e2c2-40e8-aa88-88b5f5265614
2024-11-09T02:20:36,747 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/54f3333e9615:0, corePoolSize=1, maxPoolSize=1
2024-11-09T02:20:36,749 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118866748
2024-11-09T02:20:36,750 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-09T02:20:36,751 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-09T02:20:36,751 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-09T02:20:36,752 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-11-09T02:20:36,753 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-11-09T02:20:36,754 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-09T02:20:36,755 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-09T02:20:36,755 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-09T02:20:36,755 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-09T02:20:36,756 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:36,757 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T02:20:36,757 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-09T02:20:36,757 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-09T02:20:36,757 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-09T02:20:36,760 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-09T02:20:36,761 DEBUG [RS:0;54f3333e9615:37407 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ba7ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:36,762 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-09T02:20:36,762 DEBUG [RS:0;54f3333e9615:37407 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9c699f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=54f3333e9615/172.17.0.2:0
2024-11-09T02:20:36,763 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-09T02:20:36,764 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-09T02:20:36,766 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-09T02:20:36,767 INFO  [RS:0;54f3333e9615:37407 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-11-09T02:20:36,767 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-09T02:20:36,767 INFO  [RS:0;54f3333e9615:37407 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-11-09T02:20:36,767 DEBUG [RS:0;54f3333e9615:37407 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-11-09T02:20:36,768 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741831_1007 (size=1039)
2024-11-09T02:20:36,769 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/54f3333e9615:0:becomeActiveMaster-HFileCleaner.large.0-1731118836768,5,FailOnTimeoutGroup]
2024-11-09T02:20:36,769 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/54f3333e9615:0:becomeActiveMaster-HFileCleaner.small.0-1731118836769,5,FailOnTimeoutGroup]
2024-11-09T02:20:36,769 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:36,770 INFO  [RS:0;54f3333e9615:37407 {}] regionserver.HRegionServer(3073): reportForDuty to master=54f3333e9615,40627,1731118833561 with isa=54f3333e9615/172.17.0.2:37407, startcode=1731118834294
2024-11-09T02:20:36,770 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-09T02:20:36,771 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:36,772 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:36,782 DEBUG [RS:0;54f3333e9615:37407 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-09T02:20:36,815 INFO  [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43347, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-09T02:20:36,820 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40627 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 54f3333e9615,37407,1731118834294
2024-11-09T02:20:36,823 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40627 {}] master.ServerManager(486): Registering regionserver=54f3333e9615,37407,1731118834294
2024-11-09T02:20:36,838 DEBUG [RS:0;54f3333e9615:37407 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91
2024-11-09T02:20:36,838 DEBUG [RS:0;54f3333e9615:37407 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:42041
2024-11-09T02:20:36,838 DEBUG [RS:0;54f3333e9615:37407 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-11-09T02:20:36,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T02:20:36,843 DEBUG [RS:0;54f3333e9615:37407 {}] zookeeper.ZKUtil(111): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/54f3333e9615,37407,1731118834294
2024-11-09T02:20:36,843 WARN  [RS:0;54f3333e9615:37407 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T02:20:36,843 INFO  [RS:0;54f3333e9615:37407 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-09T02:20:36,844 DEBUG [RS:0;54f3333e9615:37407 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/WALs/54f3333e9615,37407,1731118834294
2024-11-09T02:20:36,845 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [54f3333e9615,37407,1731118834294]
2024-11-09T02:20:36,857 DEBUG [RS:0;54f3333e9615:37407 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-11-09T02:20:36,868 INFO  [RS:0;54f3333e9615:37407 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-09T02:20:36,883 INFO  [RS:0;54f3333e9615:37407 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-09T02:20:36,886 INFO  [RS:0;54f3333e9615:37407 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-09T02:20:36,886 INFO  [RS:0;54f3333e9615:37407 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:36,887 INFO  [RS:0;54f3333e9615:37407 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-11-09T02:20:36,894 INFO  [RS:0;54f3333e9615:37407 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:36,894 DEBUG [RS:0;54f3333e9615:37407 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1
2024-11-09T02:20:36,894 DEBUG [RS:0;54f3333e9615:37407 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1
2024-11-09T02:20:36,894 DEBUG [RS:0;54f3333e9615:37407 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1
2024-11-09T02:20:36,895 DEBUG [RS:0;54f3333e9615:37407 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1
2024-11-09T02:20:36,895 DEBUG [RS:0;54f3333e9615:37407 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1
2024-11-09T02:20:36,895 DEBUG [RS:0;54f3333e9615:37407 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/54f3333e9615:0, corePoolSize=2, maxPoolSize=2
2024-11-09T02:20:36,895 DEBUG [RS:0;54f3333e9615:37407 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1
2024-11-09T02:20:36,895 DEBUG [RS:0;54f3333e9615:37407 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1
2024-11-09T02:20:36,895 DEBUG [RS:0;54f3333e9615:37407 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1
2024-11-09T02:20:36,895 DEBUG [RS:0;54f3333e9615:37407 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1
2024-11-09T02:20:36,896 DEBUG [RS:0;54f3333e9615:37407 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1
2024-11-09T02:20:36,896 DEBUG [RS:0;54f3333e9615:37407 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/54f3333e9615:0, corePoolSize=3, maxPoolSize=3
2024-11-09T02:20:36,896 DEBUG [RS:0;54f3333e9615:37407 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/54f3333e9615:0, corePoolSize=3, maxPoolSize=3
2024-11-09T02:20:36,897 INFO  [RS:0;54f3333e9615:37407 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:36,897 INFO  [RS:0;54f3333e9615:37407 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:36,897 INFO  [RS:0;54f3333e9615:37407 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:36,898 INFO  [RS:0;54f3333e9615:37407 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:36,898 INFO  [RS:0;54f3333e9615:37407 {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,37407,1731118834294-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T02:20:36,919 INFO  [RS:0;54f3333e9615:37407 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-09T02:20:36,921 INFO  [RS:0;54f3333e9615:37407 {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,37407,1731118834294-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:36,940 INFO  [RS:0;54f3333e9615:37407 {}] regionserver.Replication(204): 54f3333e9615,37407,1731118834294 started
2024-11-09T02:20:36,940 INFO  [RS:0;54f3333e9615:37407 {}] regionserver.HRegionServer(1767): Serving as 54f3333e9615,37407,1731118834294, RpcServer on 54f3333e9615/172.17.0.2:37407, sessionid=0x1018ad491d00001
2024-11-09T02:20:36,941 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-09T02:20:36,941 DEBUG [RS:0;54f3333e9615:37407 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 54f3333e9615,37407,1731118834294
2024-11-09T02:20:36,942 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '54f3333e9615,37407,1731118834294'
2024-11-09T02:20:36,942 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-09T02:20:36,943 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-09T02:20:36,943 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-09T02:20:36,943 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-09T02:20:36,944 DEBUG [RS:0;54f3333e9615:37407 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 54f3333e9615,37407,1731118834294
2024-11-09T02:20:36,944 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '54f3333e9615,37407,1731118834294'
2024-11-09T02:20:36,944 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-09T02:20:36,944 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-09T02:20:36,945 DEBUG [RS:0;54f3333e9615:37407 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-09T02:20:36,945 INFO  [RS:0;54f3333e9615:37407 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-09T02:20:36,945 INFO  [RS:0;54f3333e9615:37407 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-09T02:20:37,051 INFO  [RS:0;54f3333e9615:37407 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-11-09T02:20:37,055 INFO  [RS:0;54f3333e9615:37407 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54f3333e9615%2C37407%2C1731118834294, suffix=, logDir=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/WALs/54f3333e9615,37407,1731118834294, archiveDir=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/oldWALs, maxLogs=32
2024-11-09T02:20:37,072 DEBUG [RS:0;54f3333e9615:37407 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/WALs/54f3333e9615,37407,1731118834294/54f3333e9615%2C37407%2C1731118834294.1731118837057, exclude list is [], retry=0
2024-11-09T02:20:37,078 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34107,DS-e2fd8557-6f9d-43d5-826f-6bbf1bb53ac1,DISK]
2024-11-09T02:20:37,081 INFO  [RS:0;54f3333e9615:37407 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/WALs/54f3333e9615,37407,1731118834294/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:20:37,082 DEBUG [RS:0;54f3333e9615:37407 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46621:46621)]
2024-11-09T02:20:37,171 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-11-09T02:20:37,171 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91
2024-11-09T02:20:37,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741833_1009 (size=32)
2024-11-09T02:20:37,582 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T02:20:37,585 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T02:20:37,588 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T02:20:37,588 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T02:20:37,589 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T02:20:37,589 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T02:20:37,592 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T02:20:37,592 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T02:20:37,592 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T02:20:37,593 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T02:20:37,595 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T02:20:37,595 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T02:20:37,596 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T02:20:37,597 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740
2024-11-09T02:20:37,598 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740
2024-11-09T02:20:37,601 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead.
2024-11-09T02:20:37,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-11-09T02:20:37,607 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T02:20:37,609 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65270332, jitterRate=-0.02739626169204712}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242}
2024-11-09T02:20:37,612 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-11-09T02:20:37,612 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-11-09T02:20:37,613 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-11-09T02:20:37,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-11-09T02:20:37,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T02:20:37,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T02:20:37,614 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-11-09T02:20:37,614 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-11-09T02:20:37,617 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-11-09T02:20:37,617 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-11-09T02:20:37,623 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-09T02:20:37,632 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T02:20:37,634 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-09T02:20:37,786 DEBUG [54f3333e9615:40627 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-09T02:20:37,791 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=54f3333e9615,37407,1731118834294
2024-11-09T02:20:37,796 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 54f3333e9615,37407,1731118834294, state=OPENING
2024-11-09T02:20:37,801 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-09T02:20:37,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:37,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:37,805 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T02:20:37,805 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T02:20:37,806 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=54f3333e9615,37407,1731118834294}]
2024-11-09T02:20:37,981 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:37,983 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-09T02:20:37,987 INFO  [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40780, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-09T02:20:37,997 INFO  [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-11-09T02:20:37,997 INFO  [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-09T02:20:37,998 INFO  [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta
2024-11-09T02:20:38,001 INFO  [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54f3333e9615%2C37407%2C1731118834294.meta, suffix=.meta, logDir=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/WALs/54f3333e9615,37407,1731118834294, archiveDir=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/oldWALs, maxLogs=32
2024-11-09T02:20:38,018 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/WALs/54f3333e9615,37407,1731118834294/54f3333e9615%2C37407%2C1731118834294.meta.1731118838003.meta, exclude list is [], retry=0
2024-11-09T02:20:38,021 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34107,DS-e2fd8557-6f9d-43d5-826f-6bbf1bb53ac1,DISK]
2024-11-09T02:20:38,024 INFO  [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/WALs/54f3333e9615,37407,1731118834294/54f3333e9615%2C37407%2C1731118834294.meta.1731118838003.meta
2024-11-09T02:20:38,025 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46621:46621)]
2024-11-09T02:20:38,025 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-09T02:20:38,027 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-09T02:20:38,088 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-09T02:20:38,093 INFO  [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-09T02:20:38,097 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-09T02:20:38,097 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T02:20:38,097 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-11-09T02:20:38,098 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-11-09T02:20:38,101 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T02:20:38,103 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T02:20:38,103 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T02:20:38,104 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T02:20:38,104 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T02:20:38,105 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T02:20:38,106 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T02:20:38,106 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T02:20:38,107 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T02:20:38,108 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T02:20:38,108 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T02:20:38,109 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T02:20:38,110 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740
2024-11-09T02:20:38,113 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740
2024-11-09T02:20:38,116 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead.
2024-11-09T02:20:38,118 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-11-09T02:20:38,120 INFO  [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64480766, jitterRate=-0.03916171193122864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242}
2024-11-09T02:20:38,122 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-11-09T02:20:38,129 INFO  [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731118837976
2024-11-09T02:20:38,140 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-09T02:20:38,140 INFO  [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-11-09T02:20:38,141 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,37407,1731118834294
2024-11-09T02:20:38,143 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 54f3333e9615,37407,1731118834294, state=OPEN
2024-11-09T02:20:38,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T02:20:38,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T02:20:38,148 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T02:20:38,148 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T02:20:38,153 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-11-09T02:20:38,153 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=54f3333e9615,37407,1731118834294 in 342 msec
2024-11-09T02:20:38,158 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-11-09T02:20:38,158 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 531 msec
2024-11-09T02:20:38,163 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.4790 sec
2024-11-09T02:20:38,164 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731118838163, completionTime=-1
2024-11-09T02:20:38,164 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-09T02:20:38,164 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-11-09T02:20:38,205 DEBUG [hconnection-0x59d9354-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T02:20:38,208 INFO  [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40784, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T02:20:38,219 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-11-09T02:20:38,219 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118898219
2024-11-09T02:20:38,219 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118958219
2024-11-09T02:20:38,219 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 54 msec
2024-11-09T02:20:38,242 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,40627,1731118833561-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:38,242 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,40627,1731118833561-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:38,243 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,40627,1731118833561-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:38,244 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-54f3333e9615:40627, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:38,244 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:38,249 DEBUG [master/54f3333e9615:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-11-09T02:20:38,252 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-11-09T02:20:38,254 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-09T02:20:38,260 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-11-09T02:20:38,263 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-11-09T02:20:38,264 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T02:20:38,266 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-09T02:20:38,276 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741835_1011 (size=358)
2024-11-09T02:20:38,680 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 3f031be53384114a19e4a66a40a56128, NAME => 'hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91
2024-11-09T02:20:38,690 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741836_1012 (size=42)
2024-11-09T02:20:39,091 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T02:20:39,092 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 3f031be53384114a19e4a66a40a56128, disabling compactions & flushes
2024-11-09T02:20:39,092 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128.
2024-11-09T02:20:39,092 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128.
2024-11-09T02:20:39,092 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128. after waiting 0 ms
2024-11-09T02:20:39,092 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128.
2024-11-09T02:20:39,092 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128.
2024-11-09T02:20:39,092 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 3f031be53384114a19e4a66a40a56128:

2024-11-09T02:20:39,094 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-11-09T02:20:39,101 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731118839095"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118839095"}]},"ts":"1731118839095"}
2024-11-09T02:20:39,124 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-11-09T02:20:39,126 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-09T02:20:39,129 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118839126"}]},"ts":"1731118839126"}
2024-11-09T02:20:39,134 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-11-09T02:20:39,140 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=3f031be53384114a19e4a66a40a56128, ASSIGN}]
2024-11-09T02:20:39,142 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=3f031be53384114a19e4a66a40a56128, ASSIGN
2024-11-09T02:20:39,144 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=3f031be53384114a19e4a66a40a56128, ASSIGN; state=OFFLINE, location=54f3333e9615,37407,1731118834294; forceNewPlan=false, retain=false
2024-11-09T02:20:39,294 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=3f031be53384114a19e4a66a40a56128, regionState=OPENING, regionLocation=54f3333e9615,37407,1731118834294
2024-11-09T02:20:39,298 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 3f031be53384114a19e4a66a40a56128, server=54f3333e9615,37407,1731118834294}]
2024-11-09T02:20:39,452 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:39,459 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128.
2024-11-09T02:20:39,459 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 3f031be53384114a19e4a66a40a56128, NAME => 'hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128.', STARTKEY => '', ENDKEY => ''}
2024-11-09T02:20:39,460 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 3f031be53384114a19e4a66a40a56128
2024-11-09T02:20:39,460 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T02:20:39,460 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 3f031be53384114a19e4a66a40a56128
2024-11-09T02:20:39,460 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 3f031be53384114a19e4a66a40a56128
2024-11-09T02:20:39,463 INFO  [StoreOpener-3f031be53384114a19e4a66a40a56128-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3f031be53384114a19e4a66a40a56128 
2024-11-09T02:20:39,466 INFO  [StoreOpener-3f031be53384114a19e4a66a40a56128-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f031be53384114a19e4a66a40a56128 columnFamilyName info
2024-11-09T02:20:39,466 DEBUG [StoreOpener-3f031be53384114a19e4a66a40a56128-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T02:20:39,467 INFO  [StoreOpener-3f031be53384114a19e4a66a40a56128-1 {}] regionserver.HStore(327): Store=3f031be53384114a19e4a66a40a56128/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T02:20:39,469 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/namespace/3f031be53384114a19e4a66a40a56128
2024-11-09T02:20:39,469 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/namespace/3f031be53384114a19e4a66a40a56128
2024-11-09T02:20:39,473 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 3f031be53384114a19e4a66a40a56128
2024-11-09T02:20:39,477 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/namespace/3f031be53384114a19e4a66a40a56128/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T02:20:39,479 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 3f031be53384114a19e4a66a40a56128; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74073305, jitterRate=0.10377825796604156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-09T02:20:39,480 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 3f031be53384114a19e4a66a40a56128:

2024-11-09T02:20:39,482 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128., pid=6, masterSystemTime=1731118839452
2024-11-09T02:20:39,486 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128.
2024-11-09T02:20:39,486 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128.
2024-11-09T02:20:39,487 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=3f031be53384114a19e4a66a40a56128, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,37407,1731118834294
2024-11-09T02:20:39,498 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-11-09T02:20:39,499 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 3f031be53384114a19e4a66a40a56128, server=54f3333e9615,37407,1731118834294 in 195 msec
2024-11-09T02:20:39,502 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-11-09T02:20:39,502 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=3f031be53384114a19e4a66a40a56128, ASSIGN in 359 msec
2024-11-09T02:20:39,503 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-09T02:20:39,504 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118839503"}]},"ts":"1731118839503"}
2024-11-09T02:20:39,507 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-11-09T02:20:39,511 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-11-09T02:20:39,514 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2570 sec
2024-11-09T02:20:39,565 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-11-09T02:20:39,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-11-09T02:20:39,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:39,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:39,596 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-11-09T02:20:39,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-11-09T02:20:39,616 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 23 msec
2024-11-09T02:20:39,620 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-11-09T02:20:39,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-11-09T02:20:39,635 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 14 msec
2024-11-09T02:20:39,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-11-09T02:20:39,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-11-09T02:20:39,649 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 5.284sec
2024-11-09T02:20:39,650 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-09T02:20:39,652 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-09T02:20:39,653 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-09T02:20:39,653 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-09T02:20:39,653 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-09T02:20:39,654 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,40627,1731118833561-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T02:20:39,654 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,40627,1731118833561-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-09T02:20:39,661 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-11-09T02:20:39,662 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-09T02:20:39,663 INFO  [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,40627,1731118833561-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T02:20:39,750 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7502c059 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6ea82e8f
2024-11-09T02:20:39,750 WARN  [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry
2024-11-09T02:20:39,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707adb81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:39,762 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-09T02:20:39,762 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-09T02:20:39,772 DEBUG [hconnection-0x3e1871-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T02:20:39,782 INFO  [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34200, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T02:20:39,808 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=54f3333e9615,40627,1731118833561
2024-11-09T02:20:39,811 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x57d89e0d to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d7b22aa
2024-11-09T02:20:39,819 DEBUG [ReadOnlyZKClient-127.0.0.1:60825@0x57d89e0d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dae715a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:39,841 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=257, ProcessCount=11, AvailableMemoryMB=10500
2024-11-09T02:20:39,881 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching master stub from registry
2024-11-09T02:20:39,882 DEBUG [ReadOnlyZKClient-127.0.0.1:60825@0x57d89e0d {}] client.AsyncConnectionImpl(310): The fetched master address is 54f3333e9615,40627,1731118833561
2024-11-09T02:20:39,885 DEBUG [ReadOnlyZKClient-127.0.0.1:60825@0x57d89e0d {}] client.ConnectionUtils(581): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@291cd466
2024-11-09T02:20:39,886 DEBUG [ReadOnlyZKClient-127.0.0.1:60825@0x57d89e0d {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-09T02:20:39,889 INFO  [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47856, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-09T02:20:39,890 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:20:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:20:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-11-09T02:20:39,918 INFO  [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:39,920 INFO  [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:39,923 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:39,923 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00004 connected
2024-11-09T02:20:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-11-09T02:20:40,027 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00004, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:40,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:40,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T02:20:40,047 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:40,210 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:40,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=10
2024-11-09T02:20:40,212 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-09T02:20:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-11-09T02:20:40,229 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=10}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:40,250 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:40,263 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-09T02:20:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=10
2024-11-09T02:20:40,268 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:40,271 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-11-09T02:20:40,271 INFO  [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-09T02:20:40,272 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 221 msec
2024-11-09T02:20:40,274 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 380 msec
2024-11-09T02:20:40,294 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4765f97b to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@41876f6
2024-11-09T02:20:40,298 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:40,298 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:40,299 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@256ed389, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:40,301 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:40,303 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:40,304 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:40,304 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00006 connected
2024-11-09T02:20:40,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-11-09T02:20:40,536 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:20:40,539 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:20:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=11, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11
2024-11-09T02:20:40,560 INFO  [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:40,561 INFO  [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:40,564 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:40,565 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00007 connected
2024-11-09T02:20:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11
2024-11-09T02:20:40,666 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00007, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:40,671 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:40,825 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:40,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=12
2024-11-09T02:20:40,825 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:40,827 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:40,827 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:40,827 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:40,828 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00006, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:40,828 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00006, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11
2024-11-09T02:20:40,931 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00006, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:40,931 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=12}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4765f97b to 127.0.0.1:60825
2024-11-09T02:20:40,932 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=12}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:40,948 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:40,948 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:40,948 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-09T02:20:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=12
2024-11-09T02:20:40,952 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:40,955 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=12, resume processing ppid=11
2024-11-09T02:20:40,955 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, ppid=11, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 281 msec
2024-11-09T02:20:40,955 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:20:40,957 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 415 msec
2024-11-09T02:20:40,984 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6338503f to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@12df9488
2024-11-09T02:20:40,990 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:40,991 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:40,991 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@670f731c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:40,993 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:40,996 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:40,996 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:40,996 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00009 connected
2024-11-09T02:20:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11
2024-11-09T02:20:41,176 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:41,180 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:41,182 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:41,183 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:41,186 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false
2024-11-09T02:20:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=13, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13
2024-11-09T02:20:41,204 INFO  [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:41,205 INFO  [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:41,208 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:41,208 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0000a connected
2024-11-09T02:20:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13
2024-11-09T02:20:41,311 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0000a, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:41,318 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:41,471 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:41,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=14
2024-11-09T02:20:41,472 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:41,474 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:41,474 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:41,474 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:41,474 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00009, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:41,474 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00009, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13
2024-11-09T02:20:41,577 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00009, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:41,577 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=14}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6338503f to 127.0.0.1:60825
2024-11-09T02:20:41,577 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=14}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:41,592 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:41,593 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:41,593 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-09T02:20:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=14
2024-11-09T02:20:41,597 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:41,600 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13
2024-11-09T02:20:41,600 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 279 msec
2024-11-09T02:20:41,600 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false
2024-11-09T02:20:41,602 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 414 msec
2024-11-09T02:20:41,623 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x697e1dd8 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@330dae50
2024-11-09T02:20:41,627 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:41,628 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:41,628 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@593306dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:41,629 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:41,632 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:41,632 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:41,632 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0000c connected
2024-11-09T02:20:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13
2024-11-09T02:20:41,816 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:41,817 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:41,818 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:41,819 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:41,820 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:41,821 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false
2024-11-09T02:20:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15
2024-11-09T02:20:41,839 INFO  [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:41,840 INFO  [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:41,843 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:41,844 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0000d connected
2024-11-09T02:20:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15
2024-11-09T02:20:41,945 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0000d, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:41,950 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:42,103 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:42,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=16
2024-11-09T02:20:42,104 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:42,106 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:42,106 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:42,106 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:42,106 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0000c, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:42,106 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0000c, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15
2024-11-09T02:20:42,209 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0000c, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:42,209 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=16}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x697e1dd8 to 127.0.0.1:60825
2024-11-09T02:20:42,209 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=16}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:42,225 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:42,225 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:42,225 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16
2024-11-09T02:20:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=16
2024-11-09T02:20:42,229 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:42,233 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15
2024-11-09T02:20:42,233 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false
2024-11-09T02:20:42,233 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 279 msec
2024-11-09T02:20:42,235 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 412 msec
2024-11-09T02:20:42,257 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0e98b73e to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4e981ba9
2024-11-09T02:20:42,260 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:42,261 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:42,261 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1f7dc80e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:42,262 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:42,266 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:42,266 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:42,266 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0000f connected
2024-11-09T02:20:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15
2024-11-09T02:20:42,456 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:42,456 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:42,458 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:42,459 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:42,459 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:20:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=17, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17
2024-11-09T02:20:42,477 INFO  [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:42,478 INFO  [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:42,481 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:42,481 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00010 connected
2024-11-09T02:20:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17
2024-11-09T02:20:42,583 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00010, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:42,588 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:42,740 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:42,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=18
2024-11-09T02:20:42,742 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:42,743 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:42,743 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:42,743 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:42,743 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0000f, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:42,744 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0000f, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17
2024-11-09T02:20:42,846 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0000f, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:42,846 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=18}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0e98b73e to 127.0.0.1:60825
2024-11-09T02:20:42,846 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=18}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:42,861 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:42,861 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:42,862 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-09T02:20:42,862 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18
2024-11-09T02:20:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=18
2024-11-09T02:20:42,863 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-11-09T02:20:42,865 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:42,868 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=18, resume processing ppid=17
2024-11-09T02:20:42,868 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, ppid=17, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 278 msec
2024-11-09T02:20:42,868 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:20:42,870 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 409 msec
2024-11-09T02:20:42,891 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ed104d9 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@390269a2
2024-11-09T02:20:42,895 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:42,895 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:42,896 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@22dcafb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:42,897 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:42,899 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:42,900 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00012 connected
2024-11-09T02:20:42,900 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17
2024-11-09T02:20:43,086 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:43,086 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:43,087 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:43,088 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false
2024-11-09T02:20:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=19, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19
2024-11-09T02:20:43,109 INFO  [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:43,110 INFO  [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:43,112 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:43,113 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00013 connected
2024-11-09T02:20:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19
2024-11-09T02:20:43,215 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00013, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:43,221 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:43,373 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:43,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=20
2024-11-09T02:20:43,374 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:43,376 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:43,376 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:43,376 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:43,376 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00012, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:43,376 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00012, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19
2024-11-09T02:20:43,480 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00012, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:43,480 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=20}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ed104d9 to 127.0.0.1:60825
2024-11-09T02:20:43,480 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=20}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:43,495 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:43,495 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:43,495 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20
2024-11-09T02:20:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=20
2024-11-09T02:20:43,499 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:43,502 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19
2024-11-09T02:20:43,502 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false
2024-11-09T02:20:43,502 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 278 msec
2024-11-09T02:20:43,504 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 413 msec
2024-11-09T02:20:43,524 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x15371322 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8bf2131
2024-11-09T02:20:43,528 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:43,529 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:43,529 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5a4e7336, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:43,530 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:43,533 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:43,533 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:43,533 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00015 connected
2024-11-09T02:20:43,608 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-09T02:20:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19
2024-11-09T02:20:43,716 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:43,716 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:43,717 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:20:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21
2024-11-09T02:20:43,734 INFO  [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:43,735 INFO  [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:43,738 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:43,738 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00016 connected
2024-11-09T02:20:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21
2024-11-09T02:20:43,840 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00016, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:43,845 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:43,997 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:43,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=22
2024-11-09T02:20:43,998 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:44,000 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:44,000 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:44,000 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:44,000 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00015, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:44,000 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00015, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21
2024-11-09T02:20:44,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-09T02:20:44,065 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-09T02:20:44,067 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace
2024-11-09T02:20:44,067 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer
2024-11-09T02:20:44,068 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T02:20:44,068 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-09T02:20:44,068 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-09T02:20:44,068 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-09T02:20:44,103 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00015, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:44,103 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=22}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x15371322 to 127.0.0.1:60825
2024-11-09T02:20:44,103 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=22}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:44,119 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:44,120 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:44,120 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22
2024-11-09T02:20:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=22
2024-11-09T02:20:44,123 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:44,126 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21
2024-11-09T02:20:44,126 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 278 msec
2024-11-09T02:20:44,126 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:20:44,129 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 410 msec
2024-11-09T02:20:44,149 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2897b07a to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7bb6abb4
2024-11-09T02:20:44,153 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:44,153 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:44,154 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@24408674, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:44,155 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:44,158 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:44,158 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:44,158 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00018 connected
2024-11-09T02:20:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21
2024-11-09T02:20:44,346 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:44,346 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:44,349 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:20:44,349 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:20:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=23, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:20:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23
2024-11-09T02:20:44,355 DEBUG [PEWorker-1 {}] procedure.MasterProcedureScheduler(583): PEER '1', shared lock count=0, exclusively locked by procId=23
2024-11-09T02:20:44,355 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1470): LOCK_EVENT_WAIT pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:44,357 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-09T02:20:44,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-09T02:20:44,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:20:44,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:20:44,361 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=23, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:20:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23
2024-11-09T02:20:44,513 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:44,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=25
2024-11-09T02:20:44,514 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-09T02:20:44,514 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0
2024-11-09T02:20:44,514 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user
2024-11-09T02:20:44,514 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:44,514 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00018, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:44,514 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00018, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:44,617 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00018, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:44,617 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=25}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2897b07a to 127.0.0.1:60825
2024-11-09T02:20:44,617 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=25}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:44,618 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSourceManager(430): Done with the queue 1
2024-11-09T02:20:44,619 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=25}] zookeeper.ZKUtil(111): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:20:44,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:20:44,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:20:44,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:20:44,623 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=25}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:20:44,623 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=25
2024-11-09T02:20:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=25
2024-11-09T02:20:44,626 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:44,629 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=23
2024-11-09T02:20:44,629 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=23, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 265 msec
2024-11-09T02:20:44,632 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:20:44,632 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1
2024-11-09T02:20:44,634 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 282 msec
2024-11-09T02:20:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:20:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23
2024-11-09T02:20:44,675 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:20:44,676 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:20:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=26, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26
2024-11-09T02:20:44,678 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:44,684 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=26, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:44,685 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=26, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec
2024-11-09T02:20:44,686 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26
2024-11-09T02:20:44,786 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:20:44,787 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:44,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:44,790 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x1018ad491d00019 connected
2024-11-09T02:20:44,819 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=223 (was 222)
Potentially hanging thread: RSProcedureDispatcher-pool-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS_REFRESH_PEER-regionserver/54f3333e9615:0-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: ForkJoinPool.commonPool-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: RS-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=452 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=236 (was 257), ProcessCount=11 (was 11), AvailableMemoryMB=10467 (was 10500)
2024-11-09T02:20:44,827 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=223, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=236, ProcessCount=11, AvailableMemoryMB=10467
2024-11-09T02:20:44,828 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:20:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:20:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27
2024-11-09T02:20:44,846 INFO  [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:44,847 INFO  [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:44,850 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:44,850 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0001a connected
2024-11-09T02:20:44,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27
2024-11-09T02:20:44,952 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0001a, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:44,961 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:44,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:20:45,113 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:45,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=28
2024-11-09T02:20:45,114 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-09T02:20:45,130 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=28}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:45,144 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27
2024-11-09T02:20:45,155 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28
2024-11-09T02:20:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=28
2024-11-09T02:20:45,159 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:45,162 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27
2024-11-09T02:20:45,162 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-09T02:20:45,162 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 198 msec
2024-11-09T02:20:45,164 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 334 msec
2024-11-09T02:20:45,184 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x076ef6cf to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7386b87a
2024-11-09T02:20:45,188 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:45,188 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:45,188 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5edb9342, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:45,189 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:45,192 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:45,193 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:45,193 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0001c connected
2024-11-09T02:20:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27
2024-11-09T02:20:45,456 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:20:45,457 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:20:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=29, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29
2024-11-09T02:20:45,474 INFO  [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:45,475 INFO  [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:45,477 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:45,478 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0001d connected
2024-11-09T02:20:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:20:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29
2024-11-09T02:20:45,579 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0001d, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:45,584 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:45,736 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:45,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=30
2024-11-09T02:20:45,737 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:45,738 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:45,738 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:45,738 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:45,739 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0001c, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:45,739 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0001c, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29
2024-11-09T02:20:45,841 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0001c, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:45,841 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=30}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x076ef6cf to 127.0.0.1:60825
2024-11-09T02:20:45,841 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=30}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:45,856 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:45,856 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:45,857 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30
2024-11-09T02:20:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=30
2024-11-09T02:20:45,860 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:45,863 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=30, resume processing ppid=29
2024-11-09T02:20:45,863 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:20:45,863 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, ppid=29, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 276 msec
2024-11-09T02:20:45,865 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 406 msec
2024-11-09T02:20:45,886 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63dad703 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e8f045e
2024-11-09T02:20:45,889 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:45,890 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:45,890 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7ea63432, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:45,891 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:45,894 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:45,894 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:45,894 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0001f connected
2024-11-09T02:20:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29
2024-11-09T02:20:46,086 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:46,086 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:46,087 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-11-09T02:20:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=31, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31
2024-11-09T02:20:46,105 INFO  [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:46,106 INFO  [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:46,108 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:46,109 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00020 connected
2024-11-09T02:20:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31
2024-11-09T02:20:46,210 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00020, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:46,215 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:46,368 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:46,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=32
2024-11-09T02:20:46,369 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:46,370 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:46,370 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:46,370 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:46,371 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0001f, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:46,371 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0001f, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31
2024-11-09T02:20:46,473 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0001f, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:46,473 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=32}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x63dad703 to 127.0.0.1:60825
2024-11-09T02:20:46,473 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=32}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:46,488 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:46,488 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:46,488 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32
2024-11-09T02:20:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=32
2024-11-09T02:20:46,492 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:46,495 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31
2024-11-09T02:20:46,495 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 277 msec
2024-11-09T02:20:46,495 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-11-09T02:20:46,497 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 407 msec
2024-11-09T02:20:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:20:46,517 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5ae828fc to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1efc5770
2024-11-09T02:20:46,521 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:46,521 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:46,522 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@69b89bc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:46,523 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:46,525 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:46,526 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:46,526 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00022 connected
2024-11-09T02:20:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31
2024-11-09T02:20:46,716 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:46,716 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:46,717 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false
2024-11-09T02:20:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33
2024-11-09T02:20:46,735 INFO  [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:46,735 INFO  [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:46,738 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:46,738 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00023 connected
2024-11-09T02:20:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33
2024-11-09T02:20:46,840 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00023, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:46,840 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:46,842 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=33, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:46,843 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=33, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=125 msec
2024-11-09T02:20:46,843 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33
2024-11-09T02:20:47,037 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
2024-11-09T02:20:47,037 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:47,038 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false
2024-11-09T02:20:47,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=34, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:47,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34
2024-11-09T02:20:47,056 INFO  [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:47,057 INFO  [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:47,060 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:47,060 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00024 connected
2024-11-09T02:20:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34
2024-11-09T02:20:47,162 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00024, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:47,167 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:47,319 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:47,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=35
2024-11-09T02:20:47,320 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:47,321 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:47,321 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:47,321 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:47,322 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00022, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:47,322 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00022, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34
2024-11-09T02:20:47,424 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00022, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:47,424 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=35}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5ae828fc to 127.0.0.1:60825
2024-11-09T02:20:47,424 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=35}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:47,439 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:47,439 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:47,439 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35
2024-11-09T02:20:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=35
2024-11-09T02:20:47,442 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:47,445 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34
2024-11-09T02:20:47,445 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec
2024-11-09T02:20:47,445 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false
2024-11-09T02:20:47,447 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 407 msec
2024-11-09T02:20:47,467 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x049b4770 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@35d2e1dc
2024-11-09T02:20:47,470 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:47,470 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:47,471 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@10074e0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:47,471 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:47,474 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:47,475 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:47,475 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00026 connected
2024-11-09T02:20:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34
2024-11-09T02:20:47,666 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:47,666 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:47,667 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false
2024-11-09T02:20:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36
2024-11-09T02:20:47,684 INFO  [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:47,684 INFO  [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:47,687 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:47,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00027 connected
2024-11-09T02:20:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36
2024-11-09T02:20:47,789 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00027, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:47,789 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:47,790 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=36, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:47,791 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=36, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=123 msec
2024-11-09T02:20:47,792 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36
2024-11-09T02:20:47,986 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
2024-11-09T02:20:47,987 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:20:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=37, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37
2024-11-09T02:20:47,991 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-09T02:20:47,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-09T02:20:47,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:20:47,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:20:47,995 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37
2024-11-09T02:20:48,147 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:48,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=38
2024-11-09T02:20:48,148 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-09T02:20:48,148 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0
2024-11-09T02:20:48,148 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user
2024-11-09T02:20:48,148 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:48,148 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00026, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:48,148 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00026, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:48,250 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00026, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:48,250 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=38}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x049b4770 to 127.0.0.1:60825
2024-11-09T02:20:48,250 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=38}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:48,251 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSourceManager(430): Done with the queue 1
2024-11-09T02:20:48,252 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=38}] zookeeper.ZKUtil(111): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:20:48,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:20:48,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:20:48,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:20:48,255 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=38}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:20:48,255 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=38
2024-11-09T02:20:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=38
2024-11-09T02:20:48,258 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:48,261 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37
2024-11-09T02:20:48,261 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 264 msec
2024-11-09T02:20:48,264 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:20:48,264 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1
2024-11-09T02:20:48,265 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 277 msec
2024-11-09T02:20:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37
2024-11-09T02:20:48,306 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:20:48,306 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:20:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39
2024-11-09T02:20:48,308 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:48,309 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=39, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:48,310 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=39, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-09T02:20:48,310 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39
2024-11-09T02:20:48,416 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-09T02:20:48,417 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:20:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=40, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40
2024-11-09T02:20:48,420 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:48,421 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=40, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:48,422 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=40, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-09T02:20:48,422 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:20:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40
2024-11-09T02:20:48,526 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:20:48,540 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=223 (was 223), OpenFileDescriptor=452 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=236 (was 236), ProcessCount=11 (was 11), AvailableMemoryMB=10436 (was 10467)
2024-11-09T02:20:48,548 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=223, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=236, ProcessCount=11, AvailableMemoryMB=10435
2024-11-09T02:20:48,550 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:20:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=41, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:20:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41
2024-11-09T02:20:48,567 INFO  [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:48,567 INFO  [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:48,570 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:48,571 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00028 connected
2024-11-09T02:20:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41
2024-11-09T02:20:48,672 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00028, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:48,679 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:48,832 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:48,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=42
2024-11-09T02:20:48,833 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-09T02:20:48,850 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=42}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:48,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41
2024-11-09T02:20:48,871 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:48,878 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=42
2024-11-09T02:20:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=42
2024-11-09T02:20:48,881 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:48,883 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=42, resume processing ppid=41
2024-11-09T02:20:48,884 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, ppid=41, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 202 msec
2024-11-09T02:20:48,884 INFO  [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-09T02:20:48,885 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 334 msec
2024-11-09T02:20:48,905 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x77a9f45c to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@794bc5f0
2024-11-09T02:20:48,909 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:48,909 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:48,909 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@18f19f8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:48,910 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:48,912 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:48,912 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:48,912 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0002a connected
2024-11-09T02:20:49,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41
2024-11-09T02:20:49,176 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:20:49,176 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:20:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=43, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:20:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43
2024-11-09T02:20:49,193 INFO  [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:49,194 INFO  [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:49,199 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:49,200 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0002b connected
2024-11-09T02:20:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43
2024-11-09T02:20:49,301 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0002b, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:49,301 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:49,303 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=43, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:49,305 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=43, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=127 msec
2024-11-09T02:20:49,305 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43
2024-11-09T02:20:49,496 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists
2024-11-09T02:20:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-11-09T02:20:49,503 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:20:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=44, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=44
2024-11-09T02:20:49,506 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:49,507 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=44, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:49,508 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=44, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-09T02:20:49,508 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=44
2024-11-09T02:20:49,616 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:20:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-11-09T02:20:49,617 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=127.0.0.1:60825:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:20:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:20:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45
2024-11-09T02:20:49,635 INFO  [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:49,636 INFO  [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:49,639 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:49,639 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0002c connected
2024-11-09T02:20:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45
2024-11-09T02:20:49,741 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0002c, quorum=127.0.0.1:60825, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:49,748 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:49,900 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:49,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=46
2024-11-09T02:20:49,902 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=ADD_PEER
2024-11-09T02:20:49,913 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis
2024-11-09T02:20:49,917 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=46}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:49,930 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSource(229): queueId=2, ReplicationSource: 2, currentBandwidth=0
2024-11-09T02:20:49,935 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46
2024-11-09T02:20:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45
2024-11-09T02:20:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=46
2024-11-09T02:20:49,938 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for ADD on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:49,941 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45
2024-11-09T02:20:49,941 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 2, config clusterKey=127.0.0.1:60825:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-09T02:20:49,941 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 190 msec
2024-11-09T02:20:49,942 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 324 msec
2024-11-09T02:20:49,965 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2efe06c7 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@19c7ae0
2024-11-09T02:20:49,969 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,2 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:49,969 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,2 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:49,969 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6f903aa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:49,970 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 2 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:49,973 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 20x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:49,973 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,2 {}] regionserver.ReplicationSource(569): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:49,973 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 2-0x1018ad491d0002e connected
2024-11-09T02:20:50,120 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-09T02:20:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45
2024-11-09T02:20:50,246 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 2 completed
2024-11-09T02:20:50,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-11-09T02:20:50,248 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:20:50,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=47, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:50,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47
2024-11-09T02:20:50,252 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-09T02:20:50,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-09T02:20:50,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:20:50,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:20:50,256 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:50,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47
2024-11-09T02:20:50,408 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:50,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=48
2024-11-09T02:20:50,409 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-09T02:20:50,409 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0
2024-11-09T02:20:50,409 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user
2024-11-09T02:20:50,409 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:50,410 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0002a, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:50,410 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0002a, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:50,512 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0002a, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:50,512 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=48}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x77a9f45c to 127.0.0.1:60825
2024-11-09T02:20:50,512 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=48}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:50,512 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(430): Done with the queue 1
2024-11-09T02:20:50,514 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=48}] zookeeper.ZKUtil(111): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:20:50,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:20:50,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:20:50,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:20:50,517 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=48}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:20:50,517 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48
2024-11-09T02:20:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=48
2024-11-09T02:20:50,520 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:50,522 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=48, resume processing ppid=47
2024-11-09T02:20:50,522 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, ppid=47, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 264 msec
2024-11-09T02:20:50,527 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:20:50,527 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1
2024-11-09T02:20:50,529 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 279 msec
2024-11-09T02:20:50,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47
2024-11-09T02:20:50,566 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:20:50,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-11-09T02:20:50,567 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:20:50,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=49, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:50,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49
2024-11-09T02:20:50,571 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state
2024-11-09T02:20:50,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state
2024-11-09T02:20:50,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2
2024-11-09T02:20:50,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2
2024-11-09T02:20:50,575 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49
2024-11-09T02:20:50,727 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:50,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=50
2024-11-09T02:20:50,727 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=REMOVE_PEER
2024-11-09T02:20:50,727 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 2: 0
2024-11-09T02:20:50,727 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSource(657): peerId=2, Closing source 2 because: Replication stream was removed by a user
2024-11-09T02:20:50,728 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,2 {}] regionserver.ReplicationSource(508): peerId=2, Interrupted while sleeping between retries
2024-11-09T02:20:50,728 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,2 {}] zookeeper.ZKUtil(165): connection to cluster: 2-0x1018ad491d0002e, quorum=127.0.0.1:60825, baseZNode=/hbase-test2 Unable to set watcher on znode (/hbase-test2/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:50,728 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,2 {}] zookeeper.ZKWatcher(734): connection to cluster: 2-0x1018ad491d0002e, quorum=127.0.0.1:60825, baseZNode=/hbase-test2 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:50,830 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 2-0x1018ad491d0002e, quorum=127.0.0.1:60825, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:50,830 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=50}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2efe06c7 to 127.0.0.1:60825
2024-11-09T02:20:50,830 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=50}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:50,830 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSourceManager(430): Done with the queue 2
2024-11-09T02:20:50,832 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=50}] zookeeper.ZKUtil(111): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/54f3333e9615,37407,1731118834294/2/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:20:50,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/2/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:20:50,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/2
2024-11-09T02:20:50,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/2
2024-11-09T02:20:50,835 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=50}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue.
2024-11-09T02:20:50,835 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=50
2024-11-09T02:20:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=50
2024-11-09T02:20:50,837 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for REMOVE on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:50,840 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49
2024-11-09T02:20:50,840 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 262 msec
2024-11-09T02:20:50,842 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue.
2024-11-09T02:20:50,842 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 2
2024-11-09T02:20:50,844 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 275 msec
2024-11-09T02:20:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49
2024-11-09T02:20:50,886 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed
2024-11-09T02:20:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-11-09T02:20:50,887 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:20:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51
2024-11-09T02:20:50,889 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:50,890 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=51, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:50,891 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=51, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:20:50,891 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51
2024-11-09T02:20:50,996 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-09T02:20:50,997 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:20:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=52, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=52
2024-11-09T02:20:50,999 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:51,000 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=52, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:51,001 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=52, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:20:51,001 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=52
2024-11-09T02:20:51,106 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:20:51,119 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=221 (was 223), OpenFileDescriptor=446 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 236), ProcessCount=11 (was 11), AvailableMemoryMB=10421 (was 10435)
2024-11-09T02:20:51,127 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=221, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=10420
2024-11-09T02:20:51,128 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:20:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=53, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:20:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53
2024-11-09T02:20:51,130 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: java.lang.ClassNotFoundException: whatever
	at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?]
	at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?]
	at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?]
	... 12 more
2024-11-09T02:20:51,131 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=53, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
Caused by: java.lang.ClassNotFoundException: whatever
	at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?]
	at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?]
	at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:51,133 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=53, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=4 msec
2024-11-09T02:20:51,133 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53
2024-11-09T02:20:51,236 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever
2024-11-09T02:20:51,240 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:20:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54
2024-11-09T02:20:51,242 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:51,243 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=54, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:51,244 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=54, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-09T02:20:51,245 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54
2024-11-09T02:20:51,346 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-09T02:20:51,346 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:20:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=55, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55
2024-11-09T02:20:51,349 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:51,350 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=55, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:51,351 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=55, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-09T02:20:51,351 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55
2024-11-09T02:20:51,455 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:20:51,468 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=221 (was 221), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=10419 (was 10420)
2024-11-09T02:20:51,476 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=221, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=10418
2024-11-09T02:20:51,477 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:20:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=56, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:20:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=56
2024-11-09T02:20:51,480 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?]
	... 13 more
2024-11-09T02:20:51,482 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=56, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:51,483 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=56, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=4 msec
2024-11-09T02:20:51,483 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=56
2024-11-09T02:20:51,586 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever
2024-11-09T02:20:51,586 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:20:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57
2024-11-09T02:20:51,589 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:51,590 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=57, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:51,591 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=57, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-09T02:20:51,591 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57
2024-11-09T02:20:51,696 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-09T02:20:51,696 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:20:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=58, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=58
2024-11-09T02:20:51,698 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:51,700 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=58, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:51,701 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=58, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-09T02:20:51,701 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=58
2024-11-09T02:20:51,806 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:20:51,818 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=221 (was 221), OpenFileDescriptor=444 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=10417 (was 10418)
2024-11-09T02:20:51,826 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=221, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=10417
2024-11-09T02:20:51,827 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:20:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=59, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:20:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59
2024-11-09T02:20:51,845 INFO  [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:51,845 INFO  [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:51,848 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:51,848 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0002f connected
2024-11-09T02:20:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59
2024-11-09T02:20:51,950 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0002f, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:51,956 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:52,108 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:52,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=60
2024-11-09T02:20:52,109 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-09T02:20:52,125 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=60}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:52,139 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59
2024-11-09T02:20:52,146 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60
2024-11-09T02:20:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=60
2024-11-09T02:20:52,149 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:52,151 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=60, resume processing ppid=59
2024-11-09T02:20:52,151 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, ppid=59, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 193 msec
2024-11-09T02:20:52,151 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-09T02:20:52,153 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 324 msec
2024-11-09T02:20:52,174 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0598a46e to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6997dd7a
2024-11-09T02:20:52,177 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:52,177 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:52,177 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7dd4b776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:52,178 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:52,180 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:52,181 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:52,181 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00031 connected
2024-11-09T02:20:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59
2024-11-09T02:20:52,456 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:20:52,457 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:20:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=61, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61
2024-11-09T02:20:52,474 INFO  [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:52,474 INFO  [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:52,477 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:52,477 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00032 connected
2024-11-09T02:20:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:20:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61
2024-11-09T02:20:52,579 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00032, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:52,583 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:52,735 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:52,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=62
2024-11-09T02:20:52,736 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:52,738 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:52,738 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:52,738 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:52,738 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00031, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:52,738 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00031, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61
2024-11-09T02:20:52,840 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00031, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:52,840 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=62}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0598a46e to 127.0.0.1:60825
2024-11-09T02:20:52,840 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=62}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:52,855 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:52,855 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:52,856 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62
2024-11-09T02:20:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=62
2024-11-09T02:20:52,859 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:52,861 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61
2024-11-09T02:20:52,861 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 276 msec
2024-11-09T02:20:52,861 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:20:52,862 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 404 msec
2024-11-09T02:20:52,892 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x474ab3af to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3013475d
2024-11-09T02:20:52,896 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:52,897 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:52,897 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@794fd550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:52,898 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:52,903 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:52,903 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:52,903 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00034 connected
2024-11-09T02:20:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61
2024-11-09T02:20:53,086 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:53,087 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:53,088 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false
2024-11-09T02:20:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63
2024-11-09T02:20:53,105 INFO  [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:53,106 INFO  [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:53,108 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:53,109 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00035 connected
2024-11-09T02:20:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63
2024-11-09T02:20:53,210 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00035, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:53,215 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:53,367 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:53,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=64
2024-11-09T02:20:53,368 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:53,369 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:53,369 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:53,369 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:53,369 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00034, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:53,370 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00034, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63
2024-11-09T02:20:53,472 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00034, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:53,472 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=64}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x474ab3af to 127.0.0.1:60825
2024-11-09T02:20:53,472 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=64}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:53,487 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:53,487 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:53,487 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=64
2024-11-09T02:20:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=64
2024-11-09T02:20:53,490 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:53,492 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63
2024-11-09T02:20:53,492 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec
2024-11-09T02:20:53,492 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false
2024-11-09T02:20:53,493 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 404 msec
2024-11-09T02:20:53,515 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1e4186a3 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4f0a1c6c
2024-11-09T02:20:53,519 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:53,520 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:53,520 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@dc24890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:53,521 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:53,523 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:53,524 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:53,524 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00037 connected
2024-11-09T02:20:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63
2024-11-09T02:20:53,716 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:53,716 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:53,717 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:53,718 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-11-09T02:20:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=65, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65
2024-11-09T02:20:53,735 INFO  [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:53,735 INFO  [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:53,738 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:53,738 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00038 connected
2024-11-09T02:20:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65
2024-11-09T02:20:53,840 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00038, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:53,843 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:53,995 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:53,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=66
2024-11-09T02:20:53,996 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:53,997 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:53,997 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:53,997 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:53,997 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00037, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:53,997 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00037, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65
2024-11-09T02:20:54,099 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00037, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:54,099 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=66}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1e4186a3 to 127.0.0.1:60825
2024-11-09T02:20:54,099 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=66}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:54,114 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:54,114 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:54,115 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66
2024-11-09T02:20:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=66
2024-11-09T02:20:54,117 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:54,119 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=66, resume processing ppid=65
2024-11-09T02:20:54,119 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, ppid=65, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 274 msec
2024-11-09T02:20:54,119 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-11-09T02:20:54,121 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 401 msec
2024-11-09T02:20:54,142 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6bc731bc to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2b2e2533
2024-11-09T02:20:54,146 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:54,146 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:54,146 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@f9e60be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:54,147 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:54,149 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:54,150 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:54,150 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0003a connected
2024-11-09T02:20:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65
2024-11-09T02:20:54,345 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:54,346 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:54,346 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:20:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=67, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67
2024-11-09T02:20:54,350 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-09T02:20:54,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-09T02:20:54,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:20:54,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:20:54,354 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67
2024-11-09T02:20:54,505 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:54,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=68
2024-11-09T02:20:54,506 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-09T02:20:54,507 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0
2024-11-09T02:20:54,507 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user
2024-11-09T02:20:54,507 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:54,507 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0003a, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:54,507 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0003a, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:54,609 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0003a, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:54,609 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=68}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6bc731bc to 127.0.0.1:60825
2024-11-09T02:20:54,610 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=68}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:54,610 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSourceManager(430): Done with the queue 1
2024-11-09T02:20:54,611 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=68}] zookeeper.ZKUtil(111): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:20:54,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:20:54,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:20:54,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:20:54,614 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=68}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:20:54,614 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=68
2024-11-09T02:20:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=68
2024-11-09T02:20:54,616 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:54,618 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67
2024-11-09T02:20:54,618 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 262 msec
2024-11-09T02:20:54,621 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:20:54,621 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1
2024-11-09T02:20:54,622 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 274 msec
2024-11-09T02:20:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67
2024-11-09T02:20:54,665 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:20:54,666 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:20:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69
2024-11-09T02:20:54,667 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:54,668 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=69, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:54,669 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=69, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:20:54,669 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69
2024-11-09T02:20:54,775 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-09T02:20:54,776 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:20:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=70, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:20:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=70
2024-11-09T02:20:54,778 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:20:54,779 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=70, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:20:54,780 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=70, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:20:54,780 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:20:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=70
2024-11-09T02:20:54,886 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:20:54,898 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=221 (was 221), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=208 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=10386 (was 10417)
2024-11-09T02:20:54,906 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=221, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=208, ProcessCount=11, AvailableMemoryMB=10386
2024-11-09T02:20:54,907 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:20:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=71, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:20:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71
2024-11-09T02:20:54,924 INFO  [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:54,925 INFO  [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:54,928 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:54,928 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0003b connected
2024-11-09T02:20:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71
2024-11-09T02:20:55,030 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0003b, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:55,036 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:55,188 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:55,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=72
2024-11-09T02:20:55,189 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-09T02:20:55,205 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=72}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:55,219 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71
2024-11-09T02:20:55,226 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72
2024-11-09T02:20:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=72
2024-11-09T02:20:55,229 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:55,231 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=72, resume processing ppid=71
2024-11-09T02:20:55,231 INFO  [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-09T02:20:55,231 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, ppid=71, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 193 msec
2024-11-09T02:20:55,233 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 324 msec
2024-11-09T02:20:55,256 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x50642bcf to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2cd6b832
2024-11-09T02:20:55,260 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:55,260 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:55,261 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@58222ae5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:55,262 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:55,264 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:55,264 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:55,265 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0003d connected
2024-11-09T02:20:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71
2024-11-09T02:20:55,536 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:20:55,536 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:20:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=73, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73
2024-11-09T02:20:55,554 INFO  [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:55,554 INFO  [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:55,557 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:55,557 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0003e connected
2024-11-09T02:20:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73
2024-11-09T02:20:55,659 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0003e, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:55,663 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:55,815 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:55,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=74
2024-11-09T02:20:55,816 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:55,817 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:55,817 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:55,817 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:55,817 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0003d, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:55,817 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0003d, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73
2024-11-09T02:20:55,919 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0003d, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:55,919 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=74}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x50642bcf to 127.0.0.1:60825
2024-11-09T02:20:55,919 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=74}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:55,935 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:55,935 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:55,935 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74
2024-11-09T02:20:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=74
2024-11-09T02:20:55,938 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:55,940 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73
2024-11-09T02:20:55,940 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:20:55,940 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec
2024-11-09T02:20:55,942 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 404 msec
2024-11-09T02:20:55,966 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x75abc9b3 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6ee3b3d1
2024-11-09T02:20:55,969 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:55,970 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:55,970 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@64fefc08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:55,971 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:55,973 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:55,973 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:55,974 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00040 connected
2024-11-09T02:20:56,014 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-09T02:20:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73
2024-11-09T02:20:56,166 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:56,166 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:56,167 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false
2024-11-09T02:20:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75
2024-11-09T02:20:56,185 INFO  [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:56,186 INFO  [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:56,188 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:56,189 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00041 connected
2024-11-09T02:20:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75
2024-11-09T02:20:56,290 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00041, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:56,295 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:56,448 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:56,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=76
2024-11-09T02:20:56,449 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:56,450 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:56,450 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:56,450 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:56,450 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00040, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:56,451 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00040, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75
2024-11-09T02:20:56,553 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00040, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:56,553 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=76}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x75abc9b3 to 127.0.0.1:60825
2024-11-09T02:20:56,553 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=76}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:56,568 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:56,569 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:56,569 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76
2024-11-09T02:20:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=76
2024-11-09T02:20:56,572 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:56,574 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75
2024-11-09T02:20:56,574 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 277 msec
2024-11-09T02:20:56,574 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false
2024-11-09T02:20:56,575 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 407 msec
2024-11-09T02:20:56,598 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x354851f8 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6cba3984
2024-11-09T02:20:56,601 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:56,601 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:56,601 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@71557412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:56,602 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:56,604 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:56,605 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:56,605 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00043 connected
2024-11-09T02:20:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75
2024-11-09T02:20:56,795 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:56,796 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:56,797 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:56,798 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-11-09T02:20:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=77, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77
2024-11-09T02:20:56,814 INFO  [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:56,815 INFO  [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:56,819 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:56,819 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00044 connected
2024-11-09T02:20:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77
2024-11-09T02:20:56,921 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00044, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:56,924 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:57,076 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:57,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=78
2024-11-09T02:20:57,077 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:57,078 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:57,078 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:57,078 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:57,078 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00043, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:57,079 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00043, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77
2024-11-09T02:20:57,181 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00043, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:57,181 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=78}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x354851f8 to 127.0.0.1:60825
2024-11-09T02:20:57,181 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=78}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:57,196 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:57,196 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:57,196 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78
2024-11-09T02:20:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=78
2024-11-09T02:20:57,199 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:57,200 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=78, resume processing ppid=77
2024-11-09T02:20:57,201 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, ppid=77, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec
2024-11-09T02:20:57,201 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-11-09T02:20:57,202 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 403 msec
2024-11-09T02:20:57,227 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2d1c7a26 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@256371e4
2024-11-09T02:20:57,230 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:57,230 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:57,230 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@584532d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:57,231 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:57,233 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:57,234 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:57,234 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00046 connected
2024-11-09T02:20:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77
2024-11-09T02:20:57,426 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:57,426 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:57,427 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:57,429 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-11-09T02:20:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=79, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79
2024-11-09T02:20:57,446 INFO  [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:57,446 INFO  [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:57,448 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:57,449 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00047 connected
2024-11-09T02:20:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79
2024-11-09T02:20:57,550 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00047, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:57,554 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:57,706 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:57,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=80
2024-11-09T02:20:57,707 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:57,708 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:57,708 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:57,708 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:57,708 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00046, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:57,708 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00046, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79
2024-11-09T02:20:57,810 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00046, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:57,810 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=80}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2d1c7a26 to 127.0.0.1:60825
2024-11-09T02:20:57,811 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=80}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:57,826 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:57,826 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:57,827 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80
2024-11-09T02:20:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=80
2024-11-09T02:20:57,829 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:57,831 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79
2024-11-09T02:20:57,831 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-11-09T02:20:57,831 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec
2024-11-09T02:20:57,833 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 402 msec
2024-11-09T02:20:57,857 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c18bcd7 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@49ab4f95
2024-11-09T02:20:57,861 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:57,861 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:57,862 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@57abcc65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:57,862 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:57,865 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:57,865 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:57,865 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00049 connected
2024-11-09T02:20:58,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79
2024-11-09T02:20:58,055 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:58,056 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:58,057 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:58,058 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-09T02:20:58,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81
2024-11-09T02:20:58,075 INFO  [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:58,076 INFO  [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:58,078 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:58,079 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0004a connected
2024-11-09T02:20:58,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81
2024-11-09T02:20:58,180 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0004a, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:58,184 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:58,336 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:58,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=82
2024-11-09T02:20:58,337 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:58,338 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:58,338 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:58,338 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:58,338 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00049, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:58,338 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00049, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81
2024-11-09T02:20:58,440 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00049, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:58,440 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=82}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3c18bcd7 to 127.0.0.1:60825
2024-11-09T02:20:58,441 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=82}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:58,456 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:58,456 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:58,457 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82
2024-11-09T02:20:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=82
2024-11-09T02:20:58,459 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:58,461 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81
2024-11-09T02:20:58,461 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec
2024-11-09T02:20:58,461 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-09T02:20:58,462 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 403 msec
2024-11-09T02:20:58,491 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ce1bb98 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3aefc983
2024-11-09T02:20:58,494 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:58,495 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:58,495 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@51ab8e8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:58,497 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:58,499 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:58,499 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:58,500 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0004c connected
2024-11-09T02:20:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81
2024-11-09T02:20:58,686 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:58,687 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:58,688 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:58,689 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-09T02:20:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=83, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83
2024-11-09T02:20:58,707 INFO  [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:58,708 INFO  [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:58,711 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:58,711 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0004d connected
2024-11-09T02:20:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83
2024-11-09T02:20:58,813 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0004d, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:58,817 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:58,968 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:58,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=84
2024-11-09T02:20:58,969 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:58,970 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:58,970 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:58,970 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:58,970 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0004c, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:58,971 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0004c, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83
2024-11-09T02:20:59,072 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0004c, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:59,072 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=84}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1ce1bb98 to 127.0.0.1:60825
2024-11-09T02:20:59,073 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=84}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:20:59,088 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:20:59,088 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:20:59,088 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84
2024-11-09T02:20:59,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=84
2024-11-09T02:20:59,091 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:59,092 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=84, resume processing ppid=83
2024-11-09T02:20:59,092 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-09T02:20:59,092 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, ppid=83, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec
2024-11-09T02:20:59,094 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 403 msec
2024-11-09T02:20:59,117 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6819b6b1 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72737294
2024-11-09T02:20:59,120 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:20:59,121 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:20:59,121 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1874a085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:20:59,122 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:59,124 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:59,124 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:20:59,124 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0004f connected
2024-11-09T02:20:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83
2024-11-09T02:20:59,316 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:59,316 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:59,317 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-09T02:20:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=85, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85
2024-11-09T02:20:59,335 INFO  [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:59,335 INFO  [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:59,338 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:59,338 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00050 connected
2024-11-09T02:20:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85
2024-11-09T02:20:59,440 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00050, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:59,443 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:59,595 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:59,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=86
2024-11-09T02:20:59,596 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:59,597 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86
2024-11-09T02:20:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=86
2024-11-09T02:20:59,600 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:20:59,601 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85
2024-11-09T02:20:59,601 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 157 msec
2024-11-09T02:20:59,601 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-09T02:20:59,603 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 284 msec
2024-11-09T02:20:59,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85
2024-11-09T02:20:59,635 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:20:59,636 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:59,637 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:20:59,637 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false
2024-11-09T02:20:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:20:59,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87
2024-11-09T02:20:59,656 INFO  [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:20:59,657 INFO  [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:20:59,659 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:20:59,660 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00051 connected
2024-11-09T02:20:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87
2024-11-09T02:20:59,761 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00051, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:20:59,765 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:20:59,916 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:20:59,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=88
2024-11-09T02:20:59,917 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:20:59,919 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:20:59,919 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:20:59,919 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:20:59,919 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0004f, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:59,919 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0004f, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:20:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87
2024-11-09T02:21:00,021 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0004f, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:00,021 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=88}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6819b6b1 to 127.0.0.1:60825
2024-11-09T02:21:00,022 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=88}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:00,037 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:00,037 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:21:00,038 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88
2024-11-09T02:21:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=88
2024-11-09T02:21:00,040 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:00,041 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87
2024-11-09T02:21:00,041 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec
2024-11-09T02:21:00,041 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false
2024-11-09T02:21:00,043 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 405 msec
2024-11-09T02:21:00,067 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x343a263d to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6bd73e89
2024-11-09T02:21:00,070 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:00,071 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:00,071 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@660f1b96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:00,072 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:00,074 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:00,074 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:00,074 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00053 connected
2024-11-09T02:21:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87
2024-11-09T02:21:00,266 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:21:00,266 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:00,267 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false
2024-11-09T02:21:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=89, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:21:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89
2024-11-09T02:21:00,284 INFO  [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:00,285 INFO  [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:00,287 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:00,288 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00054 connected
2024-11-09T02:21:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89
2024-11-09T02:21:00,389 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00054, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:00,395 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:00,547 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:00,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=90
2024-11-09T02:21:00,548 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:21:00,549 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:21:00,549 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:21:00,549 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:00,549 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00053, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:00,549 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00053, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89
2024-11-09T02:21:00,651 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00053, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:00,651 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=90}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x343a263d to 127.0.0.1:60825
2024-11-09T02:21:00,651 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=90}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:00,667 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:00,667 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:21:00,667 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90
2024-11-09T02:21:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=90
2024-11-09T02:21:00,670 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:00,672 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=90, resume processing ppid=89
2024-11-09T02:21:00,672 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, ppid=89, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec
2024-11-09T02:21:00,672 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false
2024-11-09T02:21:00,673 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 405 msec
2024-11-09T02:21:00,697 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x40812e2a to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3d5ee08a
2024-11-09T02:21:00,700 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:00,700 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:00,700 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@39ef4b25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:00,701 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:00,703 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:00,704 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:00,704 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00056 connected
2024-11-09T02:21:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89
2024-11-09T02:21:00,895 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:21:00,896 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:00,897 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=91, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91
2024-11-09T02:21:00,900 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:00,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:00,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:00,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:00,903 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91
2024-11-09T02:21:01,055 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:01,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=92
2024-11-09T02:21:01,056 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-09T02:21:01,056 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0
2024-11-09T02:21:01,056 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user
2024-11-09T02:21:01,056 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:01,057 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00056, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:01,057 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00056, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:01,159 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00056, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:01,159 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=92}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x40812e2a to 127.0.0.1:60825
2024-11-09T02:21:01,159 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=92}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:01,159 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSourceManager(430): Done with the queue 1
2024-11-09T02:21:01,161 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=92}] zookeeper.ZKUtil(111): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:01,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:01,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:01,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:01,163 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=92}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:01,164 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92
2024-11-09T02:21:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=92
2024-11-09T02:21:01,166 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:01,168 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91
2024-11-09T02:21:01,168 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec
2024-11-09T02:21:01,171 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:01,171 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1
2024-11-09T02:21:01,173 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 275 msec
2024-11-09T02:21:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91
2024-11-09T02:21:01,215 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:01,216 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=93, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93
2024-11-09T02:21:01,218 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:01,219 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=93, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:01,219 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=93, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:21:01,220 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93
2024-11-09T02:21:01,326 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-09T02:21:01,327 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:21:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=94, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94
2024-11-09T02:21:01,329 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:01,330 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=94, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:01,331 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=94, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:21:01,331 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94
2024-11-09T02:21:01,436 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:21:01,447 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=219 (was 221), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 208), ProcessCount=11 (was 11), AvailableMemoryMB=10205 (was 10386)
2024-11-09T02:21:01,455 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=10204
2024-11-09T02:21:01,455 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:21:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=95, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:21:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95
2024-11-09T02:21:01,472 INFO  [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:01,473 INFO  [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:01,475 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:01,476 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00057 connected
2024-11-09T02:21:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95
2024-11-09T02:21:01,577 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00057, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:01,583 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:01,734 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:01,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=96
2024-11-09T02:21:01,735 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-09T02:21:01,751 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=96}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:01,766 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:01,772 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96
2024-11-09T02:21:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=96
2024-11-09T02:21:01,774 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95
2024-11-09T02:21:01,776 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=96, resume processing ppid=95
2024-11-09T02:21:01,776 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=96, ppid=95, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec
2024-11-09T02:21:01,776 INFO  [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-09T02:21:01,777 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=95, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 320 msec
2024-11-09T02:21:01,802 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x134a8577 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6234fa2b
2024-11-09T02:21:01,805 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:01,806 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:01,806 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3833bc8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:01,807 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:01,809 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:01,809 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:01,810 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00059 connected
2024-11-09T02:21:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95
2024-11-09T02:21:02,086 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-11-09T02:21:02,090 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3938): Client=jenkins//172.17.0.2 disable replication peer, id=1
2024-11-09T02:21:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=97, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure
2024-11-09T02:21:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97
2024-11-09T02:21:02,096 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97
2024-11-09T02:21:02,247 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:02,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=98
2024-11-09T02:21:02,248 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=DISABLE_PEER
2024-11-09T02:21:02,249 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:21:02,249 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:21:02,249 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:02,249 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00059, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:02,249 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00059, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:02,351 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00059, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:02,351 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=98}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x134a8577 to 127.0.0.1:60825
2024-11-09T02:21:02,351 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=98}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:02,367 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:02,367 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:21:02,367 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98
2024-11-09T02:21:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=98
2024-11-09T02:21:02,369 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for DISABLE on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:02,371 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=98, resume processing ppid=97
2024-11-09T02:21:02,371 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=98, ppid=97, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 273 msec
2024-11-09T02:21:02,371 INFO  [PEWorker-3 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1
2024-11-09T02:21:02,372 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=97, state=SUCCESS; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 281 msec
2024-11-09T02:21:02,398 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5eeb9654 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@57a92366
2024-11-09T02:21:02,401 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:02,401 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:02,401 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@13efe077, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:02,402 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:02,404 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:02,405 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:02,405 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0005b connected
2024-11-09T02:21:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97
2024-11-09T02:21:02,405 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-11-09T02:21:02,406 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=99, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99
2024-11-09T02:21:02,409 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:02,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:02,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:02,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:02,412 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:02,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99
2024-11-09T02:21:02,564 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:02,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=100
2024-11-09T02:21:02,565 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-09T02:21:02,565 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0
2024-11-09T02:21:02,565 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user
2024-11-09T02:21:02,565 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:02,565 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0005b, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:02,565 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0005b, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:21:02,667 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0005b, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:02,667 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=100}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5eeb9654 to 127.0.0.1:60825
2024-11-09T02:21:02,667 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=100}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:02,667 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSourceManager(430): Done with the queue 1
2024-11-09T02:21:02,669 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=100}] zookeeper.ZKUtil(111): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:02,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:02,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:02,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:02,671 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=100}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:02,671 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100
2024-11-09T02:21:02,672 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:21:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=100
2024-11-09T02:21:02,674 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:02,675 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=100, resume processing ppid=99
2024-11-09T02:21:02,676 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=100, ppid=99, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 262 msec
2024-11-09T02:21:02,678 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:02,678 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1
2024-11-09T02:21:02,680 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=99, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 272 msec
2024-11-09T02:21:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99
2024-11-09T02:21:02,725 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:02,726 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=101, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=101
2024-11-09T02:21:02,727 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:02,728 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=101, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:02,729 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=101, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec
2024-11-09T02:21:02,729 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=101
2024-11-09T02:21:02,836 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-09T02:21:02,836 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:21:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=102, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102
2024-11-09T02:21:02,838 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:02,839 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=102, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:02,840 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=102, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:21:02,840 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:02,873 ERROR [LeaseRenewer:jenkins@localhost:42041 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:42041,5,FailOnTimeoutGroup] died
java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null
	at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102
2024-11-09T02:21:02,946 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:21:02,958 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=218 (was 219), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=10181 (was 10204)
2024-11-09T02:21:02,966 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=10180
2024-11-09T02:21:02,966 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=dummy_peer
2024-11-09T02:21:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=103, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=103
2024-11-09T02:21:02,968 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:02,969 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=103, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:02,970 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=103, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:21:02,970 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=103
2024-11-09T02:21:03,075 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer
2024-11-09T02:21:03,076 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=104, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104
2024-11-09T02:21:03,078 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:03,078 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=104, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:03,079 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=104, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:21:03,079 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104
2024-11-09T02:21:03,185 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-09T02:21:03,186 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:21:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=105, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=105
2024-11-09T02:21:03,188 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:03,188 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=105, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:03,189 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=105, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:21:03,189 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=105
2024-11-09T02:21:03,296 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:21:03,308 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=218 (was 218), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=10177 (was 10180)
2024-11-09T02:21:03,316 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=10176
2024-11-09T02:21:03,317 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:21:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=106, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:21:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106
2024-11-09T02:21:03,334 INFO  [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:03,335 INFO  [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:03,337 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:03,338 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0005c connected
2024-11-09T02:21:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106
2024-11-09T02:21:03,439 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0005c, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:03,447 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:03,598 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:03,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=107
2024-11-09T02:21:03,599 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-09T02:21:03,615 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=107}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:03,630 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106
2024-11-09T02:21:03,641 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107
2024-11-09T02:21:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=107
2024-11-09T02:21:03,643 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:03,644 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=107, resume processing ppid=106
2024-11-09T02:21:03,645 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-09T02:21:03,645 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=107, ppid=106, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec
2024-11-09T02:21:03,646 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=106, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 328 msec
2024-11-09T02:21:03,672 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4b8fe4b5 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e816005
2024-11-09T02:21:03,675 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:03,676 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:03,676 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@65414bc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:03,678 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:03,680 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:03,680 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:03,680 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0005e connected
2024-11-09T02:21:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106
2024-11-09T02:21:03,945 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:03,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-11-09T02:21:03,946 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=108, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:03,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108
2024-11-09T02:21:03,950 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:03,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:03,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:03,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:03,952 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108
2024-11-09T02:21:04,104 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:04,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=109
2024-11-09T02:21:04,104 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-09T02:21:04,104 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0
2024-11-09T02:21:04,104 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user
2024-11-09T02:21:04,105 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:04,105 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0005e, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:04,105 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0005e, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:04,207 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0005e, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:04,207 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=109}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4b8fe4b5 to 127.0.0.1:60825
2024-11-09T02:21:04,207 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=109}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:04,207 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSourceManager(430): Done with the queue 1
2024-11-09T02:21:04,210 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=109}] zookeeper.ZKUtil(111): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:04,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:04,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:04,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:04,213 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=109}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:04,213 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=109
2024-11-09T02:21:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=109
2024-11-09T02:21:04,215 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:04,216 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=109, resume processing ppid=108
2024-11-09T02:21:04,216 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=109, ppid=108, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec
2024-11-09T02:21:04,219 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:04,219 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1
2024-11-09T02:21:04,221 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=108, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 273 msec
2024-11-09T02:21:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108
2024-11-09T02:21:04,266 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:04,266 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=110, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110
2024-11-09T02:21:04,268 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:04,269 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=110, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:04,270 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=110, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:21:04,270 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110
2024-11-09T02:21:04,376 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-09T02:21:04,377 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:21:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=111, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=111
2024-11-09T02:21:04,378 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:04,379 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=111, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:04,380 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=111, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec
2024-11-09T02:21:04,380 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=111
2024-11-09T02:21:04,486 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:21:04,497 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=218 (was 218), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=176 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=10163 (was 10176)
2024-11-09T02:21:04,505 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=176, ProcessCount=11, AvailableMemoryMB=10162
2024-11-09T02:21:04,511 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:21:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] ipc.MetricsHBaseServer(152): Unknown exception type
java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-11-09T02:21:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] ipc.CallRunner(138): callId: 347 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:47856 deadline: 1731118924511, exception=java.io.IOException: Replication peer modification disabled
2024-11-09T02:21:04,534 WARN  [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 23 ms
java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?]
	at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)

	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 33 more
2024-11-09T02:21:04,645 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:21:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] ipc.MetricsHBaseServer(152): Unknown exception type
java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-11-09T02:21:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] ipc.CallRunner(138): callId: 348 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:47856 deadline: 1731118924645, exception=java.io.IOException: Replication peer modification disabled
2024-11-09T02:21:04,646 WARN  [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 135 ms
java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?]
	at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)

	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 33 more
2024-11-09T02:21:04,647 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions:
2024-11-09T02:21:04.536Z, java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)

2024-11-09T02:21:04.647Z, java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)


2024-11-09T02:21:04,649 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=112, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112
2024-11-09T02:21:04,650 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:04,651 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=112, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:04,652 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=112, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec
2024-11-09T02:21:04,652 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:04,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112
2024-11-09T02:21:04,755 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-09T02:21:04,756 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:21:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=113, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=113
2024-11-09T02:21:04,757 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:04,758 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=113, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:04,759 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=113, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:21:04,759 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:04,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=113
2024-11-09T02:21:04,866 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:21:04,877 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=218 (was 218), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=176 (was 176), ProcessCount=11 (was 11), AvailableMemoryMB=10158 (was 10162)
2024-11-09T02:21:04,885 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=176, ProcessCount=11, AvailableMemoryMB=10157
2024-11-09T02:21:04,886 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:21:04,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=114, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:21:04,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114
2024-11-09T02:21:04,903 INFO  [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:04,904 INFO  [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:04,906 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:04,907 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0005f connected
2024-11-09T02:21:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114
2024-11-09T02:21:05,008 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0005f, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:05,013 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:05,165 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:05,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=115
2024-11-09T02:21:05,165 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-09T02:21:05,182 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=115}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:05,196 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:05,203 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=115
2024-11-09T02:21:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=115
2024-11-09T02:21:05,205 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114
2024-11-09T02:21:05,207 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=115, resume processing ppid=114
2024-11-09T02:21:05,207 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=115, ppid=114, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 192 msec
2024-11-09T02:21:05,207 INFO  [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-09T02:21:05,208 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=114, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 321 msec
2024-11-09T02:21:05,232 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0270429d to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@bb0c1e6
2024-11-09T02:21:05,235 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:05,236 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:05,236 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@61ab36fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:05,237 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:05,239 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:05,239 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:05,239 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00061 connected
2024-11-09T02:21:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114
2024-11-09T02:21:05,516 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:05,516 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:05,517 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=116, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116
2024-11-09T02:21:05,520 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:05,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:05,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:05,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:05,523 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116
2024-11-09T02:21:05,674 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:05,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=117
2024-11-09T02:21:05,675 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-09T02:21:05,675 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0
2024-11-09T02:21:05,675 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user
2024-11-09T02:21:05,675 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:05,675 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00061, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:05,675 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00061, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:05,777 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00061, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:05,777 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=117}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0270429d to 127.0.0.1:60825
2024-11-09T02:21:05,777 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=117}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:05,778 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSourceManager(430): Done with the queue 1
2024-11-09T02:21:05,779 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=117}] zookeeper.ZKUtil(111): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:05,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:05,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:05,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:05,782 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=117}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:05,782 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=117
2024-11-09T02:21:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=117
2024-11-09T02:21:05,784 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:05,785 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=117, resume processing ppid=116
2024-11-09T02:21:05,785 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=117, ppid=116, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 261 msec
2024-11-09T02:21:05,787 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:05,787 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1
2024-11-09T02:21:05,788 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=116, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 270 msec
2024-11-09T02:21:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116
2024-11-09T02:21:05,835 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:05,835 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:21:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=118, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118
2024-11-09T02:21:05,837 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:05,838 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=118, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:05,838 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=118, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:21:05,838 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118
2024-11-09T02:21:05,946 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:21:05,959 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=218 (was 218), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=176 (was 176), ProcessCount=11 (was 11), AvailableMemoryMB=10144 (was 10157)
2024-11-09T02:21:05,967 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=176, ProcessCount=11, AvailableMemoryMB=10143
2024-11-09T02:21:05,968 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:21:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=119, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:21:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119
2024-11-09T02:21:05,985 INFO  [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:05,985 INFO  [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:05,988 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:05,988 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00062 connected
2024-11-09T02:21:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119
2024-11-09T02:21:06,090 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00062, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:06,095 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:06,247 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:06,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=120
2024-11-09T02:21:06,248 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-09T02:21:06,264 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=120}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:06,278 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119
2024-11-09T02:21:06,287 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120
2024-11-09T02:21:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=120
2024-11-09T02:21:06,289 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:06,291 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=120, resume processing ppid=119
2024-11-09T02:21:06,291 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=120, ppid=119, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 194 msec
2024-11-09T02:21:06,291 INFO  [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-09T02:21:06,292 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 323 msec
2024-11-09T02:21:06,316 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x71fd1675 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@163c14b5
2024-11-09T02:21:06,320 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:06,320 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:06,320 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4c5366b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:06,321 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:06,324 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:06,325 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:06,325 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00064 connected
2024-11-09T02:21:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119
2024-11-09T02:21:06,595 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:06,596 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:06,596 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false
2024-11-09T02:21:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=121, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:21:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121
2024-11-09T02:21:06,613 INFO  [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:06,614 INFO  [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:06,617 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:06,617 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00065 connected
2024-11-09T02:21:06,674 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-09T02:21:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121
2024-11-09T02:21:06,719 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00065, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:06,722 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:06,873 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:06,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=122
2024-11-09T02:21:06,874 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:21:06,875 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122
2024-11-09T02:21:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=122
2024-11-09T02:21:06,877 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:06,878 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=122, resume processing ppid=121
2024-11-09T02:21:06,879 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false
2024-11-09T02:21:06,879 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=122, ppid=121, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 155 msec
2024-11-09T02:21:06,880 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=121, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 283 msec
2024-11-09T02:21:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121
2024-11-09T02:21:06,916 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:21:06,916 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:06,917 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=123, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123
2024-11-09T02:21:06,920 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:06,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:06,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:06,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:06,922 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123
2024-11-09T02:21:07,074 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:07,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=124
2024-11-09T02:21:07,074 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-09T02:21:07,075 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0
2024-11-09T02:21:07,075 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user
2024-11-09T02:21:07,075 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:07,075 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00064, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:07,075 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00064, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:07,077 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:42041 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:42041,5,FailOnTimeoutGroup] died
java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null
	at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:07,177 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00064, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:07,177 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=124}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x71fd1675 to 127.0.0.1:60825
2024-11-09T02:21:07,177 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=124}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:07,177 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSourceManager(430): Done with the queue 1
2024-11-09T02:21:07,179 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=124}] zookeeper.ZKUtil(111): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:07,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:07,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:07,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:07,181 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=124}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:07,181 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124
2024-11-09T02:21:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=124
2024-11-09T02:21:07,183 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:07,185 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=124, resume processing ppid=123
2024-11-09T02:21:07,185 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=124, ppid=123, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 261 msec
2024-11-09T02:21:07,187 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:07,187 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1
2024-11-09T02:21:07,188 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=123, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 270 msec
2024-11-09T02:21:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123
2024-11-09T02:21:07,235 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:07,235 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=125, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125
2024-11-09T02:21:07,237 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:07,238 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=125, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:07,238 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=125, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:21:07,238 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125
2024-11-09T02:21:07,346 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-09T02:21:07,347 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:21:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=126, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126
2024-11-09T02:21:07,348 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:07,349 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=126, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:07,350 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=126, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec
2024-11-09T02:21:07,350 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126
2024-11-09T02:21:07,456 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:21:07,467 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=217 (was 218), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=176 (was 176), ProcessCount=11 (was 11), AvailableMemoryMB=10118 (was 10143)
2024-11-09T02:21:07,475 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=217, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=176, ProcessCount=11, AvailableMemoryMB=10117
2024-11-09T02:21:07,476 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:21:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=127, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:21:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127
2024-11-09T02:21:07,482 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127
2024-11-09T02:21:07,633 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:07,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=128
2024-11-09T02:21:07,634 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-09T02:21:07,664 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:07,671 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128
2024-11-09T02:21:07,672 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(587): peerId=1, queueId=1 (queues=1) is replicating from cluster=2c4c9e0c-e2c2-40e8-aa88-88b5f5265614 to cluster=2c4c9e0c-e2c2-40e8-aa88-88b5f5265614
2024-11-09T02:21:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=128
2024-11-09T02:21:07,673 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(343): peerId=1, starting shipping worker for walGroupId=54f3333e9615%2C37407%2C1731118834294
2024-11-09T02:21:07,673 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:07,675 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=128, resume processing ppid=127
2024-11-09T02:21:07,675 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-09T02:21:07,675 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=128, ppid=127, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec
2024-11-09T02:21:07,675 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSourceWALReader(112): peerClusterZnode=1, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1
2024-11-09T02:21:07,676 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=127, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 199 msec
2024-11-09T02:21:07,676 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1.replicationSource.shipper54f3333e9615%2C37407%2C1731118834294,1 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: 54f3333e9615%2C37407%2C1731118834294
2024-11-09T02:21:07,676 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1.replicationSource.wal-reader.54f3333e9615%2C37407%2C1731118834294,1 {}] regionserver.WALEntryStream(260): Creating new reader hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/WALs/54f3333e9615,37407,1731118834294/54f3333e9615%2C37407%2C1731118834294.1731118837057, startPosition=0, beingWritten=true
2024-11-09T02:21:07,744 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1.replicationSource.wal-reader.54f3333e9615%2C37407%2C1731118834294,1 {}] regionserver.ReplicationSourceWALReader(181): Read 0 WAL entries eligible for replication
2024-11-09T02:21:07,744 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1.replicationSource.wal-reader.54f3333e9615%2C37407%2C1731118834294,1 {}] regionserver.WALEntryStream(223): Reset reader hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/WALs/54f3333e9615,37407,1731118834294/54f3333e9615%2C37407%2C1731118834294.1731118837057 to pos 589, reset compression=false
2024-11-09T02:21:07,744 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1.replicationSource.shipper54f3333e9615%2C37407%2C1731118834294,1 {}] regionserver.ReplicationSourceShipper(110): Shipper from source 1 got entry batch from reader: WALEntryBatch [walEntries=[], lastWalPath=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/WALs/54f3333e9615,37407,1731118834294/54f3333e9615%2C37407%2C1731118834294.1731118837057, lastWalPosition=589, nbRowKeys=0, nbHFiles=0, heapSize=0, lastSeqIds={}, endOfFile=false,usedBufferSize=0]
2024-11-09T02:21:07,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127
2024-11-09T02:21:07,795 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:07,795 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:21:07,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=129, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:21:07,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=129
2024-11-09T02:21:07,797 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: 
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: java.io.IOException: Cluster key passed  is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?]
	... 13 more
2024-11-09T02:21:07,798 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=129, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: 
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: 
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
Caused by: java.io.IOException: Cluster key passed  is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:07,799 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=129, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec
2024-11-09T02:21:07,799 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=129
2024-11-09T02:21:07,906 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 
2024-11-09T02:21:07,906 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:07,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=130, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:07,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130
2024-11-09T02:21:07,909 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:07,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:07,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:07,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:07,912 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130
2024-11-09T02:21:08,063 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:08,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=131
2024-11-09T02:21:08,064 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-09T02:21:08,064 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0
2024-11-09T02:21:08,064 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user
2024-11-09T02:21:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130
2024-11-09T02:21:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130
2024-11-09T02:21:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130
2024-11-09T02:21:09,065 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1.replicationSource.wal-reader.54f3333e9615%2C37407%2C1731118834294,1 {}] util.Threads(127): sleep interrupted
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:131) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:166) ~[classes/:?]
2024-11-09T02:21:09,065 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1.replicationSource.shipper54f3333e9615%2C37407%2C1731118834294,1 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch
java.lang.InterruptedException: null
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?]
	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:313) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?]
2024-11-09T02:21:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130
2024-11-09T02:21:10,065 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSource(711): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1.replicationSource.shipper54f3333e9615%2C37407%2C1731118834294,1 terminated
2024-11-09T02:21:10,065 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSourceManager(430): Done with the queue 1
2024-11-09T02:21:10,066 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=131}] zookeeper.ZKUtil(111): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:10,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:10,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:10,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:10,068 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=131}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:10,068 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=131
2024-11-09T02:21:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=131
2024-11-09T02:21:10,070 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:10,071 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=131, resume processing ppid=130
2024-11-09T02:21:10,072 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=131, ppid=130, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1580 sec
2024-11-09T02:21:10,074 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:10,074 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1
2024-11-09T02:21:10,075 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=130, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.1670 sec
2024-11-09T02:21:10,764 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-09T02:21:10,766 INFO  [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48420, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-09T02:21:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130
2024-11-09T02:21:12,065 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:12,065 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-11-09T02:21:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=132, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132
2024-11-09T02:21:12,067 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
2024-11-09T02:21:12,068 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=132, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?]
	... 9 more
2024-11-09T02:21:12,069 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=132, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-09T02:21:12,069 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-09T02:21:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132
2024-11-09T02:21:12,176 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-09T02:21:12,188 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=218 (was 217) - Thread LEAK? -, OpenFileDescriptor=451 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=162 (was 176), ProcessCount=11 (was 11), AvailableMemoryMB=10100 (was 10117)
2024-11-09T02:21:12,196 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=218, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=162, ProcessCount=11, AvailableMemoryMB=10100
2024-11-09T02:21:12,198 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-09T02:21:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=133, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-09T02:21:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133
2024-11-09T02:21:12,215 INFO  [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:12,215 INFO  [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:12,218 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:12,219 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00066 connected
2024-11-09T02:21:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133
2024-11-09T02:21:12,320 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00066, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:12,326 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:12,477 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:12,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=134
2024-11-09T02:21:12,478 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-09T02:21:12,494 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=134}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:12,509 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133
2024-11-09T02:21:12,516 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134
2024-11-09T02:21:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=134
2024-11-09T02:21:12,517 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:12,519 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=134, resume processing ppid=133
2024-11-09T02:21:12,519 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=134, ppid=133, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 192 msec
2024-11-09T02:21:12,519 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-09T02:21:12,520 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=133, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 321 msec
2024-11-09T02:21:12,556 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a9ef383 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@252f35f
2024-11-09T02:21:12,559 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:12,559 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:12,560 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@cc0eb77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:12,560 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:12,563 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:12,563 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:12,563 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00068 connected
2024-11-09T02:21:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:21:12,679 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-09T02:21:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133
2024-11-09T02:21:12,826 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:12,826 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:21:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=135, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:21:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135
2024-11-09T02:21:12,843 INFO  [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:12,843 INFO  [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:12,848 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:12,848 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00069 connected
2024-11-09T02:21:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135
2024-11-09T02:21:12,949 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00069, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:12,952 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:13,103 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:13,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=136
2024-11-09T02:21:13,104 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:21:13,105 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:21:13,105 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:21:13,106 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:13,106 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00068, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:13,106 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00068, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:13,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135
2024-11-09T02:21:13,207 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00068, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:13,208 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=136}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0a9ef383 to 127.0.0.1:60825
2024-11-09T02:21:13,208 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=136}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:13,223 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:13,223 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:21:13,223 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136
2024-11-09T02:21:13,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=136
2024-11-09T02:21:13,225 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:13,226 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=136, resume processing ppid=135
2024-11-09T02:21:13,226 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=136, ppid=135, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 273 msec
2024-11-09T02:21:13,226 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:21:13,227 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=135, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 400 msec
2024-11-09T02:21:13,253 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x053adfbd to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1fe90775
2024-11-09T02:21:13,256 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:13,256 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:13,257 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5e16e0ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:13,257 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:13,260 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:13,260 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:13,260 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0006b connected
2024-11-09T02:21:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135
2024-11-09T02:21:13,455 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:21:13,456 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:13,456 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:13,457 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:13,457 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false
2024-11-09T02:21:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=137, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:21:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137
2024-11-09T02:21:13,474 INFO  [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:13,474 INFO  [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:13,477 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:13,477 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0006c connected
2024-11-09T02:21:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137
2024-11-09T02:21:13,579 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0006c, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:13,582 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:13,733 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:13,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=138
2024-11-09T02:21:13,734 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:21:13,735 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:21:13,735 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:21:13,735 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:13,735 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0006b, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:13,736 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0006b, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:13,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137
2024-11-09T02:21:13,837 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0006b, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:13,837 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=138}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x053adfbd to 127.0.0.1:60825
2024-11-09T02:21:13,838 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=138}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:13,853 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:13,853 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:21:13,853 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138
2024-11-09T02:21:13,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=138
2024-11-09T02:21:13,855 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:13,857 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=138, resume processing ppid=137
2024-11-09T02:21:13,857 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=138, ppid=137, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 274 msec
2024-11-09T02:21:13,857 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false
2024-11-09T02:21:13,858 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=137, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 400 msec
2024-11-09T02:21:13,883 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f7edcae to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@21b21ab
2024-11-09T02:21:13,886 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:13,886 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:13,886 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@a7d652b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:13,888 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:13,890 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:13,891 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:13,891 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0006e connected
2024-11-09T02:21:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137
2024-11-09T02:21:14,085 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:21:14,086 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:14,086 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:14,087 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:14,088 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:14,088 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false
2024-11-09T02:21:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=139, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:21:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139
2024-11-09T02:21:14,105 INFO  [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:14,105 INFO  [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:14,107 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:14,108 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0006f connected
2024-11-09T02:21:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139
2024-11-09T02:21:14,209 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d0006f, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:14,212 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:14,363 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:14,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=140
2024-11-09T02:21:14,364 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:21:14,365 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:21:14,365 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:21:14,365 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:14,365 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0006e, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:14,366 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0006e, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139
2024-11-09T02:21:14,468 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0006e, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:14,468 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=140}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2f7edcae to 127.0.0.1:60825
2024-11-09T02:21:14,468 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=140}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:14,483 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:14,483 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:21:14,484 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140
2024-11-09T02:21:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=140
2024-11-09T02:21:14,485 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:14,487 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=140, resume processing ppid=139
2024-11-09T02:21:14,487 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=140, ppid=139, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 273 msec
2024-11-09T02:21:14,487 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false
2024-11-09T02:21:14,510 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=139, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 399 msec
2024-11-09T02:21:14,533 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0cf71896 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@65ef23f9
2024-11-09T02:21:14,536 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:14,537 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:14,537 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@62237fbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:14,538 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:14,540 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:14,540 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:14,540 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00071 connected
2024-11-09T02:21:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139
2024-11-09T02:21:14,716 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:21:14,716 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:14,716 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:14,717 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:14,718 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:21:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=141, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:21:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141
2024-11-09T02:21:14,733 INFO  [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:14,733 INFO  [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:14,735 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:14,736 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00072 connected
2024-11-09T02:21:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141
2024-11-09T02:21:14,837 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00072, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:14,840 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:14,992 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:14,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=142
2024-11-09T02:21:14,993 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:21:14,994 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:21:14,994 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:21:14,994 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:14,994 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00071, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:14,994 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00071, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141
2024-11-09T02:21:15,096 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00071, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:15,097 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=142}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0cf71896 to 127.0.0.1:60825
2024-11-09T02:21:15,097 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=142}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:15,111 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:15,111 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:21:15,112 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142
2024-11-09T02:21:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=142
2024-11-09T02:21:15,113 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:15,115 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=142, resume processing ppid=141
2024-11-09T02:21:15,115 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=142, ppid=141, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 274 msec
2024-11-09T02:21:15,115 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:21:15,116 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=141, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 397 msec
2024-11-09T02:21:15,139 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3818e6eb to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@52574745
2024-11-09T02:21:15,142 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:15,142 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:15,142 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3c93df9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:15,143 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:15,145 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:15,146 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:15,146 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00074 connected
2024-11-09T02:21:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141
2024-11-09T02:21:15,346 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:21:15,346 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:15,347 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:15,347 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false
2024-11-09T02:21:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=143, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:21:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143
2024-11-09T02:21:15,363 INFO  [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:15,363 INFO  [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:15,366 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:15,366 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00075 connected
2024-11-09T02:21:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143
2024-11-09T02:21:15,468 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00075, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:15,471 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:15,623 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:15,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=144
2024-11-09T02:21:15,624 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:21:15,625 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:21:15,625 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:21:15,625 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:15,625 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00074, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:15,625 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00074, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143
2024-11-09T02:21:15,727 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00074, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:15,727 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=144}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3818e6eb to 127.0.0.1:60825
2024-11-09T02:21:15,727 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=144}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:15,741 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:15,742 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:21:15,742 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144
2024-11-09T02:21:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=144
2024-11-09T02:21:15,744 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:15,745 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=144, resume processing ppid=143
2024-11-09T02:21:15,745 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false
2024-11-09T02:21:15,745 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=144, ppid=143, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 273 msec
2024-11-09T02:21:15,746 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=143, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 398 msec
2024-11-09T02:21:15,769 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x283e93f0 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@32bb6155
2024-11-09T02:21:15,778 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:15,778 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:15,778 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@69069de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:15,779 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:15,785 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:15,785 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:15,785 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d00077 connected
2024-11-09T02:21:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143
2024-11-09T02:21:15,975 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:21:15,975 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:15,976 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:21:15,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=145, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-09T02:21:15,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145
2024-11-09T02:21:15,991 INFO  [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry
2024-11-09T02:21:15,992 INFO  [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:15,998 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id0x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:15,998 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00078 connected
2024-11-09T02:21:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145
2024-11-09T02:21:16,099 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3ffc75a6check-peer-cluster-id-0x1018ad491d00078, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:16,103 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:16,254 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:16,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=146
2024-11-09T02:21:16,255 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-09T02:21:16,256 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1
2024-11-09T02:21:16,256 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-09T02:21:16,256 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:16,256 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d00077, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:16,257 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d00077, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145
2024-11-09T02:21:16,358 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d00077, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:16,358 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=146}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x283e93f0 to 127.0.0.1:60825
2024-11-09T02:21:16,358 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=146}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:16,372 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0
2024-11-09T02:21:16,373 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSourceManager(383): Startup replication source for 1
2024-11-09T02:21:16,373 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146
2024-11-09T02:21:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=146
2024-11-09T02:21:16,375 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:16,376 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=146, resume processing ppid=145
2024-11-09T02:21:16,376 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:60825:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-09T02:21:16,376 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=146, ppid=145, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 272 msec
2024-11-09T02:21:16,377 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=145, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 400 msec
2024-11-09T02:21:16,400 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58861b18 to 127.0.0.1:60825 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14b9188b
2024-11-09T02:21:16,403 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed
java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionImplementation.<init>(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	... 1 more
2024-11-09T02:21:16,403 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster
2024-11-09T02:21:16,404 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@d82c9dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T02:21:16,404 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:60825
2024-11-09T02:21:16,406 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T02:21:16,407 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-09T02:21:16,407 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1018ad491d0007a connected
2024-11-09T02:21:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145
2024-11-09T02:21:16,605 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-09T02:21:16,606 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-11-09T02:21:16,606 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:16,606 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-11-09T02:21:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=147, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] procedure2.ProcedureExecutor(1098): Stored pid=148, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:16,607 DEBUG [PEWorker-1 {}] procedure.MasterProcedureScheduler(583): PEER '1', shared lock count=0, exclusively locked by procId=147
2024-11-09T02:21:16,607 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1470): LOCK_EVENT_WAIT pid=148, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-09T02:21:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147
2024-11-09T02:21:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:21:16,609 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:16,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-09T02:21:16,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:16,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40627-0x1018ad491d00000, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-09T02:21:16,612 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-09T02:21:16,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147
2024-11-09T02:21:16,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:21:16,763 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,37407,1731118834294
2024-11-09T02:21:16,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=149
2024-11-09T02:21:16,764 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-09T02:21:16,764 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0
2024-11-09T02:21:16,764 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user
2024-11-09T02:21:16,764 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries
2024-11-09T02:21:16,764 WARN  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1018ad491d0007a, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid)
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:16,764 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1018ad491d0007a, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException
java.lang.InterruptedException: null
	at java.lang.Object.wait(Native Method) ~[?:?]
	at java.lang.Object.wait(Object.java:338) ~[?:?]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:21:16,867 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1018ad491d0007a, quorum=127.0.0.1:60825, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T02:21:16,867 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=149}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x58861b18 to 127.0.0.1:60825
2024-11-09T02:21:16,867 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=149}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T02:21:16,867 INFO  [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSourceManager(430): Done with the queue 1
2024-11-09T02:21:16,872 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=149}] zookeeper.ZKUtil(111): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:16,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1/54f3333e9615%2C37407%2C1731118834294.1731118837057
2024-11-09T02:21:16,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:16,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37407-0x1018ad491d00001, quorum=127.0.0.1:60825, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/54f3333e9615,37407,1731118834294/1
2024-11-09T02:21:16,877 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=149}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:16,877 DEBUG [RS_REFRESH_PEER-regionserver/54f3333e9615:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149
2024-11-09T02:21:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.HMaster(4106): Remote procedure done, pid=149
2024-11-09T02:21:16,879 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 54f3333e9615,37407,1731118834294 suceeded
2024-11-09T02:21:16,881 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=149, resume processing ppid=147
2024-11-09T02:21:16,881 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=149, ppid=147, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 267 msec
2024-11-09T02:21:16,886 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue.
2024-11-09T02:21:16,886 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1
2024-11-09T02:21:16,886 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=147, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 280 msec
2024-11-09T02:21:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147
2024-11-09T02:21:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:21:16,926 INFO  [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-09T02:21:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:21:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:21:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:21:20,473 INFO  [master/54f3333e9615:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-09T02:21:20,473 INFO  [master/54f3333e9615:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-09T02:21:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:21:22,369 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-09T02:21:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:21:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:21:32,672 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:21:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:21:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:21:38,243 DEBUG [master/54f3333e9615:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 3f031be53384114a19e4a66a40a56128 changed from -1.0 to 0.0, refreshing cache
2024-11-09T02:21:44,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:21:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:21:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:22:02,672 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:22:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:22:13,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:22:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:22:32,672 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:22:33,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:22:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:22:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:23:02,673 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:23:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:23:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:23:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:23:32,673 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:23:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:23:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:23:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:24:02,673 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:24:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:24:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:24:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:24:32,673 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:24:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:24:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:24:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:25:02,674 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:25:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:25:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:25:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:25:32,674 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:25:34,307 INFO  [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=660.99 KB, freeSize=879.35 MB, max=880 MB, blockCount=0, accesses=0, hits=0, hitRatio=0, cachingAccesses=0, cachingHits=0, cachingHitsRatio=0,evictions=29, evicted=0, evictedPerRun=0.0
2024-11-09T02:25:34,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:25:34,368 DEBUG [master/54f3333e9615:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=0, created chunk count=7, reused chunk count=0, reuseRatio=0.00%
2024-11-09T02:25:34,368 DEBUG [master/54f3333e9615:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-11-09T02:25:36,940 INFO  [54f3333e9615:37407Replication Statistics #0 {}] regionserver.Replication$ReplicationStatisticsTask(247): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B

2024-11-09T02:25:38,246 DEBUG [hconnection-0x59d9354-shared-pool-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T02:25:38,248 INFO  [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38422, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T02:25:38,253 DEBUG [master/54f3333e9615:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-11-09T02:25:38,255 DEBUG [master/54f3333e9615:0.Chore.1 {}] balancer.BaseLoadBalancer(778): Start Generate Balance plan for cluster.
2024-11-09T02:25:38,255 DEBUG [master/54f3333e9615:0.Chore.1 {}] master.HMaster(2144): Balancer is going into sleep until next period in 300000ms
2024-11-09T02:25:39,951 INFO  [regionserver/54f3333e9615:0.Chore.1 {}] regionserver.HRegionServer$PeriodicMemStoreFlusher(2070): MemstoreFlusherChore requesting flush of hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128. because 3f031be53384114a19e4a66a40a56128/info has an old edit so flush to free WALs after random delay 17481 ms
2024-11-09T02:25:39,951 INFO  [regionserver/54f3333e9615:0.Chore.1 {}] regionserver.HRegionServer$PeriodicMemStoreFlusher(2070): MemstoreFlusherChore requesting flush of hbase:meta,,1.1588230740 because 1588230740/info has an old edit so flush to free WALs after random delay 97369 ms
2024-11-09T02:25:43,609 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-09T02:25:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:25:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:25:57,431 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 3f031be53384114a19e4a66a40a56128 1/1 column families, dataSize=78 B heapSize=488 B
2024-11-09T02:25:57,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/namespace/3f031be53384114a19e4a66a40a56128/.tmp/info/eb207d079f00434d865cf879d9e305ff is 45, key is default/info:d/1731118839604/Put/seqid=0
2024-11-09T02:25:57,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741837_1013 (size=5037)
2024-11-09T02:25:57,906 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/namespace/3f031be53384114a19e4a66a40a56128/.tmp/info/eb207d079f00434d865cf879d9e305ff
2024-11-09T02:25:57,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/namespace/3f031be53384114a19e4a66a40a56128/.tmp/info/eb207d079f00434d865cf879d9e305ff as hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/namespace/3f031be53384114a19e4a66a40a56128/info/eb207d079f00434d865cf879d9e305ff
2024-11-09T02:25:57,965 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/namespace/3f031be53384114a19e4a66a40a56128/info/eb207d079f00434d865cf879d9e305ff, entries=2, sequenceid=6, filesize=4.9 K
2024-11-09T02:25:57,967 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 3f031be53384114a19e4a66a40a56128 in 536ms, sequenceid=6, compaction requested=false
2024-11-09T02:25:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 3f031be53384114a19e4a66a40a56128:

2024-11-09T02:26:02,674 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:26:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:26:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:26:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:26:27,460 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:42041 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:42041,5,FailOnTimeoutGroup] died
java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null
	at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:26:32,674 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:26:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:26:39,461 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:26:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:26:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:27:02,674 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:27:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:27:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:27:17,320 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-09T02:27:17,320 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=1.23 KB heapSize=2.87 KB
2024-11-09T02:27:17,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740/.tmp/info/e96dac6b68254bde8d12488f20f033b7 is 143, key is hbase:namespace,,1731118838253.3f031be53384114a19e4a66a40a56128./info:regioninfo/1731118839487/Put/seqid=0
2024-11-09T02:27:17,354 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741838_1014 (size=6595)
2024-11-09T02:27:17,756 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.14 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740/.tmp/info/e96dac6b68254bde8d12488f20f033b7
2024-11-09T02:27:17,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740/.tmp/table/2398cc6c7ad74400aa5ae1b9a02c31b9 is 51, key is hbase:namespace/table:state/1731118839503/Put/seqid=0
2024-11-09T02:27:17,784 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741839_1015 (size=5242)
2024-11-09T02:27:18,185 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740/.tmp/table/2398cc6c7ad74400aa5ae1b9a02c31b9
2024-11-09T02:27:18,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740/.tmp/info/e96dac6b68254bde8d12488f20f033b7 as hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740/info/e96dac6b68254bde8d12488f20f033b7
2024-11-09T02:27:18,202 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740/info/e96dac6b68254bde8d12488f20f033b7, entries=10, sequenceid=9, filesize=6.4 K
2024-11-09T02:27:18,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740/.tmp/table/2398cc6c7ad74400aa5ae1b9a02c31b9 as hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740/table/2398cc6c7ad74400aa5ae1b9a02c31b9
2024-11-09T02:27:18,211 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/data/hbase/meta/1588230740/table/2398cc6c7ad74400aa5ae1b9a02c31b9, entries=2, sequenceid=9, filesize=5.1 K
2024-11-09T02:27:18,212 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~1.23 KB/1264, heapSize ~2.59 KB/2648, currentSize=0 B/0 for 1588230740 in 892ms, sequenceid=9, compaction requested=false
2024-11-09T02:27:18,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 1588230740:

2024-11-09T02:27:23,097 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:27:24,461 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:27:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:27:32,675 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:27:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:27:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:27:47,329 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:42041 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:42041,5,FailOnTimeoutGroup] died
java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null
	at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:27:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:28:02,675 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:28:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:28:08,097 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:28:09,462 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:28:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:28:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:28:32,675 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:28:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:28:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:28:53,097 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:28:54,462 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:28:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:29:02,675 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:29:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:29:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:29:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:29:32,675 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:29:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:29:38,098 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:29:39,462 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:29:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:29:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:30:02,676 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:30:08,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:30:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:30:23,098 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:30:24,463 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:30:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:30:32,676 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:30:34,307 INFO  [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=660.99 KB, freeSize=879.35 MB, max=880 MB, blockCount=0, accesses=0, hits=0, hitRatio=0, cachingAccesses=0, cachingHits=0, cachingHitsRatio=0,evictions=59, evicted=0, evictedPerRun=0.0
2024-11-09T02:30:34,367 DEBUG [master/54f3333e9615:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=3, created chunk count=7, reused chunk count=0, reuseRatio=0.00%
2024-11-09T02:30:34,368 DEBUG [master/54f3333e9615:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-11-09T02:30:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:30:36,758 DEBUG [master/54f3333e9615:0.Chore.1 {}] replication.ZKReplicationQueueStorage(530): Didn't find a RegionServer that replicates, won't prevent deletions.
2024-11-09T02:30:36,940 INFO  [54f3333e9615:37407Replication Statistics #0 {}] regionserver.Replication$ReplicationStatisticsTask(247): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B

2024-11-09T02:30:38,274 DEBUG [hconnection-0x59d9354-shared-pool-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T02:30:38,275 INFO  [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35490, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T02:30:38,292 DEBUG [master/54f3333e9615:0.Chore.1 {}] balancer.BaseLoadBalancer(778): Start Generate Balance plan for cluster.
2024-11-09T02:30:38,292 DEBUG [master/54f3333e9615:0.Chore.1 {}] master.HMaster(2144): Balancer is going into sleep until next period in 300000ms
2024-11-09T02:30:38,296 DEBUG [master/54f3333e9615:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-11-09T02:30:43,609 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-09T02:30:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:30:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:31:02,676 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:31:08,098 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:31:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:31:09,463 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:31:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:31:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:31:32,676 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:31:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:31:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:31:53,099 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:31:54,463 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:31:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:32:02,677 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:32:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:32:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:32:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:32:32,677 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:32:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:32:38,099 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:32:39,464 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:32:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:32:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:33:02,677 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:33:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:33:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:33:23,099 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:33:24,464 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:33:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:33:32,677 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:33:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:33:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:33:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:34:02,677 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:34:08,100 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:34:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:34:09,464 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:34:17,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:34:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:34:32,678 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:34:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:34:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:34:53,100 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:34:54,465 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:34:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:35:02,678 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:35:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:35:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:35:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:35:32,678 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:35:34,307 INFO  [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=663.48 KB, freeSize=879.35 MB, max=880 MB, blockCount=2, accesses=2, hits=0, hitRatio=0, cachingAccesses=2, cachingHits=0, cachingHitsRatio=0,evictions=89, evicted=0, evictedPerRun=0.0
2024-11-09T02:35:34,368 DEBUG [master/54f3333e9615:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=3, created chunk count=7, reused chunk count=0, reuseRatio=0.00%
2024-11-09T02:35:34,368 DEBUG [master/54f3333e9615:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-11-09T02:35:36,340 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(200): WAL AsyncFSWAL 54f3333e9615%2C40627%2C1731118833561:(num 1731118836258) roll period 900000 ms elapsed
2024-11-09T02:35:36,354 DEBUG [master:store-WAL-Roller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/WALs/54f3333e9615,40627,1731118833561/54f3333e9615%2C40627%2C1731118833561.1731119736340, exclude list is [], retry=0
2024-11-09T02:35:36,359 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34107,DS-e2fd8557-6f9d-43d5-826f-6bbf1bb53ac1,DISK]
2024-11-09T02:35:36,362 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/WALs/54f3333e9615,40627,1731118833561/54f3333e9615%2C40627%2C1731118833561.1731118836258 with entries=1161, filesize=503.14 KB; new WAL /user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/WALs/54f3333e9615,40627,1731118833561/54f3333e9615%2C40627%2C1731118833561.1731119736340
2024-11-09T02:35:36,362 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46621:46621)]
2024-11-09T02:35:36,362 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/WALs/54f3333e9615,40627,1731118833561/54f3333e9615%2C40627%2C1731118833561.1731118836258 is not closed yet, will try archiving it next time
2024-11-09T02:35:36,365 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741830_1006 (size=515222)
2024-11-09T02:35:36,454 INFO  [master:store-Flusher {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=428.68 KB heapSize=530.64 KB
2024-11-09T02:35:36,472 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1238171c3a36421795623bc230f5e963 is 82, key is hbase:meta,,1/info:regioninfo/1731118838141/Put/seqid=0
2024-11-09T02:35:36,476 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741841_1017 (size=5672)
2024-11-09T02:35:36,877 INFO  [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1164 (bloomFilter=true), to=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1238171c3a36421795623bc230f5e963
2024-11-09T02:35:36,911 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b894c36a25374afe8bc6d67ce5cbec6c is 293, key is \x00\x00\x00\x00\x00\x00\x00\x94/proc:d/1731118876607/Put/seqid=0
2024-11-09T02:35:36,916 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741842_1018 (size=11538)
2024-11-09T02:35:36,917 INFO  [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=427.13 KB at sequenceid=1164 (bloomFilter=true), to=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b894c36a25374afe8bc6d67ce5cbec6c
2024-11-09T02:35:36,925 INFO  [master:store-Flusher {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b894c36a25374afe8bc6d67ce5cbec6c
2024-11-09T02:35:36,940 INFO  [54f3333e9615:37407Replication Statistics #0 {}] regionserver.Replication$ReplicationStatisticsTask(247): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B

2024-11-09T02:35:36,947 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5e6ce9fd24434e03bbcb32cef4f9544d is 69, key is 54f3333e9615,37407,1731118834294/rs:state/1731118836825/Put/seqid=0
2024-11-09T02:35:36,952 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741843_1019 (size=5156)
2024-11-09T02:35:37,353 INFO  [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1164 (bloomFilter=true), to=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5e6ce9fd24434e03bbcb32cef4f9544d
2024-11-09T02:35:37,376 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0fe7e12e867542659c1af39f83ae4dac is 68, key is replication_peer_modification_on/state:d/1731118872177/Put/seqid=0
2024-11-09T02:35:37,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34107 is added to blk_1073741844_1020 (size=5154)
2024-11-09T02:35:37,782 INFO  [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1 KB at sequenceid=1164 (bloomFilter=true), to=hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0fe7e12e867542659c1af39f83ae4dac
2024-11-09T02:35:37,790 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1238171c3a36421795623bc230f5e963 as hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1238171c3a36421795623bc230f5e963
2024-11-09T02:35:37,796 INFO  [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1238171c3a36421795623bc230f5e963, entries=8, sequenceid=1164, filesize=5.5 K
2024-11-09T02:35:37,797 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b894c36a25374afe8bc6d67ce5cbec6c as hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b894c36a25374afe8bc6d67ce5cbec6c
2024-11-09T02:35:37,803 INFO  [master:store-Flusher {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b894c36a25374afe8bc6d67ce5cbec6c
2024-11-09T02:35:37,803 INFO  [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b894c36a25374afe8bc6d67ce5cbec6c, entries=149, sequenceid=1164, filesize=11.3 K
2024-11-09T02:35:37,805 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5e6ce9fd24434e03bbcb32cef4f9544d as hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5e6ce9fd24434e03bbcb32cef4f9544d
2024-11-09T02:35:37,810 INFO  [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5e6ce9fd24434e03bbcb32cef4f9544d, entries=1, sequenceid=1164, filesize=5.0 K
2024-11-09T02:35:37,812 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0fe7e12e867542659c1af39f83ae4dac as hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0fe7e12e867542659c1af39f83ae4dac
2024-11-09T02:35:37,818 INFO  [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0fe7e12e867542659c1af39f83ae4dac, entries=1, sequenceid=1164, filesize=5.0 K
2024-11-09T02:35:37,819 INFO  [master:store-Flusher {}] regionserver.HRegion(3040): Finished flush of dataSize ~428.68 KB/438973, heapSize ~530.58 KB/543312, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1365ms, sequenceid=1164, compaction requested=false
2024-11-09T02:35:37,819 DEBUG [master:store-Flusher {}] regionserver.HRegion(2538): Flush status journal for 1595e783b53d99cd5eef43b6debb2682:

2024-11-09T02:35:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:35:38,100 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:35:39,465 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:35:39,875 DEBUG [master/54f3333e9615:0.Chore.1 {}] balancer.BaseLoadBalancer(778): Start Generate Balance plan for cluster.
2024-11-09T02:35:39,875 DEBUG [master/54f3333e9615:0.Chore.1 {}] master.HMaster(2144): Balancer is going into sleep until next period in 300000ms
2024-11-09T02:35:39,876 DEBUG [hconnection-0x59d9354-shared-pool-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T02:35:39,877 INFO  [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56160, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T02:35:39,881 DEBUG [master/54f3333e9615:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-11-09T02:35:43,608 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-09T02:35:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:35:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:36:02,678 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:36:06,359 ERROR [LeaseRenewer:jenkins@localhost:42041 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:42041,5,FailOnTimeoutGroup] died
java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null
	at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T02:36:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:36:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:36:23,101 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:36:24,465 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:36:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:36:32,679 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:36:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:36:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:36:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:37:02,679 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:37:08,101 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:37:09,466 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:37:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:37:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:37:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:37:32,679 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:37:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:37:50,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:37:53,101 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:37:54,466 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:37:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:38:02,679 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:38:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:38:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:38:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:38:32,679 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:38:38,102 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:38:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:38:39,466 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:38:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:38:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:39:02,680 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:39:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:39:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:39:23,102 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:39:24,466 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:39:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:39:32,680 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:39:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:39:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:39:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:40:02,680 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:40:08,102 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:40:09,467 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:40:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:40:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:40:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:40:32,680 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:40:34,307 INFO  [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=663.48 KB, freeSize=879.35 MB, max=880 MB, blockCount=2, accesses=4, hits=2, hitRatio=50.00%, , cachingAccesses=4, cachingHits=2, cachingHitsRatio=50.00%, evictions=119, evicted=0, evictedPerRun=0.0
2024-11-09T02:40:34,368 DEBUG [master/54f3333e9615:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=6, created chunk count=7, reused chunk count=1, reuseRatio=12.50%
2024-11-09T02:40:34,368 DEBUG [master/54f3333e9615:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-11-09T02:40:36,940 INFO  [54f3333e9615:37407Replication Statistics #0 {}] regionserver.Replication$ReplicationStatisticsTask(247): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B

2024-11-09T02:40:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:40:40,291 DEBUG [hconnection-0x59d9354-shared-pool-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T02:40:40,292 INFO  [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42254, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T02:40:40,295 DEBUG [master/54f3333e9615:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-11-09T02:40:40,295 DEBUG [master/54f3333e9615:0.Chore.1 {}] replication.ZKReplicationQueueStorage(530): Didn't find a RegionServer that replicates, won't prevent deletions.
2024-11-09T02:40:40,304 DEBUG [master/54f3333e9615:0.Chore.1 {}] balancer.BaseLoadBalancer(778): Start Generate Balance plan for cluster.
2024-11-09T02:40:40,304 DEBUG [master/54f3333e9615:0.Chore.1 {}] master.HMaster(2144): Balancer is going into sleep until next period in 300000ms
2024-11-09T02:40:43,619 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-09T02:40:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:40:53,103 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:40:54,467 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:40:59,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:41:02,680 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:41:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:41:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:41:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:41:32,681 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:41:38,103 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:41:39,467 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:41:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:41:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:42:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:42:02,681 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:42:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:42:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:42:23,103 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:42:24,468 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:42:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:42:32,681 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:42:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:42:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:43:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:43:02,681 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:43:08,104 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:43:09,468 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:43:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:43:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:43:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:43:32,681 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:43:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:43:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:43:53,104 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:43:54,468 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:44:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:44:02,682 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:44:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:44:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:44:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:44:32,682 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:44:38,104 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:44:39,469 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:44:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:44:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:45:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:45:02,682 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:45:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:45:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:45:23,105 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:45:24,469 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:45:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:45:32,682 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:45:34,307 INFO  [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=663.48 KB, freeSize=879.35 MB, max=880 MB, blockCount=2, accesses=6, hits=4, hitRatio=66.67%, , cachingAccesses=6, cachingHits=4, cachingHitsRatio=66.67%, evictions=149, evicted=0, evictedPerRun=0.0
2024-11-09T02:45:34,367 DEBUG [master/54f3333e9615:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=6, created chunk count=7, reused chunk count=1, reuseRatio=12.50%
2024-11-09T02:45:34,368 DEBUG [master/54f3333e9615:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-11-09T02:45:36,940 INFO  [54f3333e9615:37407Replication Statistics #0 {}] regionserver.Replication$ReplicationStatisticsTask(247): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B

2024-11-09T02:45:40,783 DEBUG [hconnection-0x59d9354-shared-pool-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T02:45:40,785 INFO  [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57876, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T02:45:40,787 DEBUG [master/54f3333e9615:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-11-09T02:45:40,795 DEBUG [master/54f3333e9615:0.Chore.1 {}] balancer.BaseLoadBalancer(778): Start Generate Balance plan for cluster.
2024-11-09T02:45:40,795 DEBUG [master/54f3333e9615:0.Chore.1 {}] master.HMaster(2144): Balancer is going into sleep until next period in 300000ms
2024-11-09T02:45:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:45:43,609 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-09T02:45:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:46:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
2024-11-09T02:46:02,683 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T02:46:08,105 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837
2024-11-09T02:46:09,469 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3f031be53384114a19e4a66a40a56128, had cached 0 bytes from a total of 5037
2024-11-09T02:46:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148
2024-11-09T02:46:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24
====> TEST TIMED OUT. PRINTING THREAD DUMP. <====

Timestamp: 2024-11-09 02:46:29,933

"IPC Server handler 0 on default port 46271" daemon prio=5 tid=104 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"BootstrapNodeManager" daemon prio=5 tid=260 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"pool-1-thread-2" daemon prio=5 tid=15 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
        at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.run(EPollPort.java:275)
        at java.base@17.0.11/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:113)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp1312628392-40" daemon prio=5 tid=40 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f26504252a8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-7" daemon prio=5 tid=163 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Timer for 'HBase' metrics system" daemon prio=5 tid=838 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
        at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
"RS-EventLoopGroup-1-2" daemon prio=10 tid=267 runnable
java.lang.Thread.State: RUNNABLE
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"region-location-2" daemon prio=5 tid=744 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=40627" daemon prio=5 tid=182 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"NIOWorkerThread-13" daemon prio=5 tid=169 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RedundancyMonitor" daemon prio=5 tid=47 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at java.base@17.0.11/java.lang.Thread.sleep(Thread.java:344)
        at java.base@17.0.11/java.util.concurrent.TimeUnit.sleep(TimeUnit.java:446)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$RedundancyMonitor.run(BlockManager.java:5352)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOServerCxnFactory.SelectorThread-1" daemon prio=5 tid=139 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
"IPC Server handler 3 on default port 42041" daemon prio=5 tid=67 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"RS-EventLoopGroup-3-1" daemon prio=10 tid=186 runnable
java.lang.Thread.State: RUNNABLE
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"master/54f3333e9615:0.Chore.1" daemon prio=5 tid=258 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Time-limited test-EventThread" daemon prio=5 tid=193 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
"NIOServerCxnFactory.SelectorThread-0" daemon prio=5 tid=138 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
"org.apache.hadoop.hdfs.PeerCache@7eab35b1" daemon prio=5 tid=215 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
        at app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
        at app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Idle-Rpc-Conn-Sweeper-pool-0" daemon prio=5 tid=242 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp1312628392-37" daemon prio=5 tid=37 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f26504252a8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"pool-1-thread-1" daemon prio=5 tid=14 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.poll(EPollPort.java:200)
        at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.run(EPollPort.java:281)
        at java.base@17.0.11/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:113)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@b364b1a" daemon prio=5 tid=49 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:563)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-1" daemon prio=5 tid=145 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcClient-timer-pool-0" daemon prio=5 tid=241 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
        at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"PacketResponder: BP-956810084-172.17.0.2-1731118830705:blk_1073741832_1008, type=LAST_IN_PIPELINE" daemon prio=5 tid=283 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.Object.wait(Object.java:338)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0" daemon prio=5 tid=312 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"java.util.concurrent.ThreadPoolExecutor$Worker@6613a4ff[State = -1, empty queue]" daemon prio=5 tid=135 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"normalizer-worker-0" daemon prio=5 tid=252 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at app//org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorkQueue.take(RegionNormalizerWorkQueue.java:146)
        at app//org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorker.run(RegionNormalizerWorker.java:191)
        at java.base@17.0.11/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Finalizer" daemon prio=8 tid=3 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
        at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
        at java.base@17.0.11/java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:172)
"process reaper" daemon prio=10 tid=835 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/cluster_b0eb3e89-6ba9-0dea-c5b3-6b36556d49a8/dfs/data/data1/current/BP-956810084-172.17.0.2-1731118830705" daemon prio=5 tid=124 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"PEWorker-2" daemon prio=5 tid=246 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149)
        at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2128)
"qtp1965036093-89" daemon prio=5 tid=89 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-12" daemon prio=5 tid=168 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:60825@0x7502c059" daemon prio=5 tid=320 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$759/0x00007f2650b69000.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40627" daemon prio=5 tid=175 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"zk-event-processor-pool-0" daemon prio=5 tid=160 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40627" daemon prio=5 tid=174 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=5 tid=23 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
        at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
        at app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp1312628392-39" daemon prio=5 tid=39 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f26504252a8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ProcExecTimeout" daemon prio=5 tid=243 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:279)
        at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81)
        at app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56)
"SSL Certificates Store Monitor" daemon prio=5 tid=25 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.Object.wait(Object.java:338)
        at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
        at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
"regionserver/54f3333e9615:0.procedureResultReporter" daemon prio=5 tid=276 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)
"pool-14-thread-1" daemon prio=5 tid=132 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"MemStoreFlusher.0" daemon prio=5 tid=277 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:77)
        at app//org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:322)
"RS_REFRESH_PEER-regionserver/54f3333e9615:0-0" daemon prio=5 tid=334 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-4" daemon prio=5 tid=159 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server idle connection scanner for port 42041" daemon prio=5 tid=56 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
        at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
"qtp1312628392-41-acceptor-0@5a8e340a-ServerConnector@1f7e2804{HTTP/1.1, (http/1.1)}{localhost:45143}" daemon prio=3 tid=41 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method)
        at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
        at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-3" daemon prio=5 tid=158 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"regionserver/54f3333e9615:0.Chore.1" daemon prio=5 tid=273 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"PacketResponder: BP-956810084-172.17.0.2-1731118830705:blk_1073741840_1016, type=LAST_IN_PIPELINE" daemon prio=5 tid=793 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.Object.wait(Object.java:338)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"OldWALsCleaner-0" daemon prio=5 tid=257 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner.deleteFile(LogCleaner.java:172)
        at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner.lambda$createOldWalsCleaner$1(LogCleaner.java:152)
        at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner$$Lambda$587/0x00007f2650a738e0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"surefire-forkedjvm-command-thread" daemon prio=5 tid=18 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at java.base@17.0.11/sun.nio.ch.PendingFuture.get(PendingFuture.java:178)
        at app//org.apache.maven.surefire.api.util.internal.Channels$2.read(Channels.java:127)
        at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
        at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
        at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
        at app//org.apache.maven.surefire.api.util.internal.Channels$3.readImpl(Channels.java:169)
        at app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleReadableChannel.read(AbstractNoninterruptibleReadableChannel.java:50)
        at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.read(AbstractStreamDecoder.java:430)
        at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.read(AbstractStreamDecoder.java:419)
        at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.readMessageType(AbstractStreamDecoder.java:116)
        at app//org.apache.maven.surefire.booter.stream.CommandDecoder.decode(CommandDecoder.java:77)
        at app//org.apache.maven.surefire.booter.spi.CommandChannelDecoder.decode(CommandChannelDecoder.java:60)
        at app//org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:290)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"CacheReplicationMonitor(1894465686)" daemon prio=5 tid=75 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:186)
"master/54f3333e9615:0:becomeActiveMaster-MemStoreChunkPool Statistics" daemon prio=5 tid=213 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"AsyncFSWAL-0-hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91-prefix:54f3333e9615,37407,1731118834294" daemon prio=5 tid=284 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RS-EventLoopGroup-1-1" daemon prio=10 tid=154 runnable
java.lang.Thread.State: RUNNABLE
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"master/54f3333e9615:0:becomeActiveMaster-HFileCleaner.small.0-1731118836769" daemon prio=5 tid=266 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535)
        at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner.consumerLoop(HFileCleaner.java:285)
        at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner$2.run(HFileCleaner.java:269)
"IPC Server handler 1 on default port 46271" daemon prio=5 tid=105 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"Time-limited test-EventThread" daemon prio=5 tid=156 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
"Monitor thread for TaskMonitor" daemon prio=5 tid=209 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp1965036093-90" daemon prio=5 tid=90 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Session-HouseKeeper-67bf5572-1"  prio=5 tid=45 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Time-limited test.named-queue-events-pool-0" daemon prio=5 tid=152 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
        at app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
        at app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
        at app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"MarkedDeleteBlockScrubberThread" daemon prio=5 tid=48 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$MarkedDeleteBlockScrubber.run(BlockManager.java:5326)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"master/54f3333e9615:0:becomeActiveMaster-MemStoreChunkPool Statistics" daemon prio=5 tid=211 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"master:store-WAL-Roller" daemon prio=5 tid=225 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:179)
"NIOWorkerThread-10" daemon prio=5 tid=166 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"54f3333e9615:40627" daemon prio=5 tid=251 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager.waitOnAssignQueue(AssignmentManager.java:2370)
        at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager.processAssignQueue(AssignmentManager.java:2392)
        at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager.access$600(AssignmentManager.java:112)
        at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager$1.run(AssignmentManager.java:2332)
"IPC Client (1877429482) connection to localhost/127.0.0.1:42041 from jenkins" daemon prio=5 tid=109 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
        at app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
"pool-5-thread-1"  prio=5 tid=36 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=40627" daemon prio=5 tid=181 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"HBase-Metrics2-1" daemon prio=5 tid=153 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Block report processor" daemon prio=5 tid=51 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.processQueue(BlockManager.java:5627)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.run(BlockManager.java:5614)
"zk-event-processor-pool-0" daemon prio=5 tid=194 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp1312628392-38" daemon prio=5 tid=38 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f26504252a8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"nioEventLoopGroup-2-1"  prio=10 tid=92 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
        at app//io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
        at app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
        at app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
        at app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37407" daemon prio=5 tid=196 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"PEWorker-5" daemon prio=5 tid=249 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149)
        at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2128)
"IPC Server handler 2 on default port 42041" daemon prio=5 tid=66 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"SessionTracker" daemon prio=5 tid=141 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)
"RequestThrottler" daemon prio=5 tid=144 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
"IPC Server Responder" daemon prio=5 tid=97 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733)
        at app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716)
"NIOServerCxnFactory.AcceptThread:localhost/127.0.0.1:60825" daemon prio=5 tid=140 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
"main"  prio=5 tid=1 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/java.lang.Thread.dumpThreads(Native Method)
        at java.base@17.0.11/java.lang.Thread.getAllStackTraces(Thread.java:1671)
        at app//org.apache.hadoop.hbase.TimedOutTestsListener.buildThreadDump(TimedOutTestsListener.java:92)
        at app//org.apache.hadoop.hbase.TimedOutTestsListener.buildThreadDiagnosticString(TimedOutTestsListener.java:78)
        at app//org.apache.hadoop.hbase.TimedOutTestsListener.testFailure(TimedOutTestsListener.java:65)
        at app//org.junit.runner.notification.SynchronizedRunListener.testFailure(SynchronizedRunListener.java:94)
        at app//org.junit.runner.notification.RunNotifier$6.notifyListener(RunNotifier.java:177)
        at app//org.junit.runner.notification.RunNotifier$SafeNotifier.run(RunNotifier.java:72)
        at app//org.junit.runner.notification.RunNotifier.fireTestFailures(RunNotifier.java:173)
        at app//org.junit.runner.notification.RunNotifier.fireTestFailure(RunNotifier.java:167)
        at app//org.apache.maven.surefire.common.junit4.Notifier.fireTestFailure(Notifier.java:100)
        at app//org.junit.internal.runners.model.EachTestNotifier.addFailure(EachTestNotifier.java:23)
        at app//org.junit.internal.runners.model.EachTestNotifier.addMultipleFailureException(EachTestNotifier.java:29)
        at app//org.junit.internal.runners.model.EachTestNotifier.addFailure(EachTestNotifier.java:21)
        at app//org.junit.runners.ParentRunner.run(ParentRunner.java:419)
        at app//org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:316)
        at app//org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:240)
        at app//org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:214)
        at app//org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:155)
        at app//org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:385)
        at app//org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162)
        at app//org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:507)
        at app//org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:495)
"Async-Client-Retry-Timer-pool-0" daemon prio=5 tid=319 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
        at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"JvmPauseMonitor" daemon prio=5 tid=271 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hbase.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:148)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"DataXceiver for client DFSClient_NONMAPREDUCE_-1728371911_22 at /127.0.0.1:44054 [Receiving block BP-956810084-172.17.0.2-1731118830705:blk_1073741832_1008]" daemon prio=5 tid=282 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
        at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
        at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
        at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
        at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
        at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151)
        at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Hadoop-Metrics-Updater-0" daemon prio=5 tid=98 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"pool-11-thread-1"  prio=5 tid=69 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@262e079b" daemon prio=5 tid=74 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:4689)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"MobFileCache #0" daemon prio=5 tid=191 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-11" daemon prio=5 tid=167 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"master:store-Flusher" daemon prio=5 tid=240 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759)
        at app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor.flushLoop(MasterRegionFlusherAndCompactor.java:200)
        at app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor$$Lambda$480/0x00007f26509d8d00.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-2" daemon prio=5 tid=157 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"pool-17-thread-1"  prio=5 tid=86 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=40627" daemon prio=5 tid=179 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"RS_OPEN_META-regionserver/54f3333e9615:0-0" daemon prio=5 tid=294 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@6b49abe4" daemon prio=5 tid=72 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:4550)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.util.JvmPauseMonitor$Monitor@df7e1a0" daemon prio=5 tid=93 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:60825@0x57d89e0d" daemon prio=5 tid=325 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$759/0x00007f2650b69000.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Signal Dispatcher" daemon prio=9 tid=4 runnable
java.lang.Thread.State: RUNNABLE
"IPC Server handler 3 on default port 46271" daemon prio=5 tid=107 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/cluster_b0eb3e89-6ba9-0dea-c5b3-6b36556d49a8/dfs/data/data2/current/BP-956810084-172.17.0.2-1731118830705" daemon prio=5 tid=123 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"SyncThread:0" daemon prio=5 tid=142 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
"regionserver/54f3333e9615:0.leaseChecker" daemon prio=5 tid=274 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)
"NIOWorkerThread-14" daemon prio=5 tid=170 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"SnapshotHandlerChoreCleaner" daemon prio=5 tid=253 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RS-EventLoopGroup-1-3" daemon prio=10 tid=329 runnable
java.lang.Thread.State: RUNNABLE
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"PEWorker-1" daemon prio=5 tid=245 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149)
        at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2128)
"Common-Cleaner" daemon prio=8 tid=12 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
        at java.base@17.0.11/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:140)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
        at java.base@17.0.11/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:162)
"RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37407" daemon prio=5 tid=200 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@6132670e" daemon prio=5 tid=71 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:537)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp1312628392-42" daemon prio=5 tid=42 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RS:0;54f3333e9615:37407-longCompactions-0" daemon prio=5 tid=272 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at app//org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:101)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"pool-19-thread-1"  prio=5 tid=103 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-8" daemon prio=5 tid=164 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"LruBlockCacheStatsExecutor" daemon prio=5 tid=189 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"AsyncFSWAL-0-hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91-prefix:54f3333e9615,37407,1731118834294.meta" daemon prio=5 tid=297 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"PacketResponder: BP-956810084-172.17.0.2-1731118830705:blk_1073741834_1010, type=LAST_IN_PIPELINE" daemon prio=5 tid=296 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.Object.wait(Object.java:338)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server idle connection scanner for port 46271" daemon prio=5 tid=96 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
        at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
"org.apache.hadoop.util.JvmPauseMonitor$Monitor@7eb48b85" daemon prio=5 tid=34 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Time-limited test-SendThread(127.0.0.1:60825)" daemon prio=5 tid=403 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
"RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40627" daemon prio=5 tid=176 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"zk-event-processor-pool-0" daemon prio=5 tid=405 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40627" daemon prio=5 tid=173 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40627" daemon prio=5 tid=177 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/cluster_b0eb3e89-6ba9-0dea-c5b3-6b36556d49a8/dfs/data/data2)" daemon prio=5 tid=118 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656)
"IPC Server listener on 0" daemon prio=5 tid=94 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559)
"Notification Thread" daemon prio=9 tid=13 runnable
java.lang.Thread.State: RUNNABLE
"qtp1965036093-87" daemon prio=5 tid=87 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f26504252a8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RPCClient-NioEventLoopGroup-4-1" daemon prio=5 tid=324 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ConnnectionExpirer" daemon prio=5 tid=137 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$ConnectionExpirerThread.run(NIOServerCnxnFactory.java:554)
"RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=37407" daemon prio=5 tid=204 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/df0a0bc5-ac06-91e8-f0ad-e305527b85c6/cluster_b0eb3e89-6ba9-0dea-c5b3-6b36556d49a8/dfs/data/data1)" daemon prio=5 tid=117 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656)
"Session-HouseKeeper-3685d53b-1"  prio=5 tid=91 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37407" daemon prio=5 tid=197 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"IPC Server handler 4 on default port 42041" daemon prio=5 tid=68 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"Hadoop-Metrics-Updater-0" daemon prio=5 tid=58 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor@5ec1d4f9" daemon prio=5 tid=61 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor.run(PendingReconstructionBlocks.java:267)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RPCClient-NioEventLoopGroup-4-2" daemon prio=5 tid=328 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"region-location-1" daemon prio=5 tid=318 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"54f3333e9615:37407Replication Statistics #0" daemon prio=5 tid=279 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=37407" daemon prio=5 tid=202 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"DatanodeAdminMonitor-0" daemon prio=5 tid=62 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Socket Reader #1 for port 0" daemon prio=5 tid=95 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497)
        at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476)
"RS-EventLoopGroup-3-2" daemon prio=10 tid=234 runnable
java.lang.Thread.State: RUNNABLE
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37407" daemon prio=5 tid=198 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"WorkerMonitor" daemon prio=5 tid=244 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:279)
        at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81)
        at app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56)
"AsyncFSWAL-0-hdfs://localhost:42041/user/jenkins/test-data/c0c2b3ce-dac7-c963-2746-3bb0b2d22a91/MasterData-prefix:54f3333e9615,40627,1731118833561" daemon prio=5 tid=237 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RS_COMPACTED_FILES_DISCHARGER-regionserver/54f3333e9615:0-0" daemon prio=5 tid=747 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-5" daemon prio=5 tid=161 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-6" daemon prio=5 tid=162 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=37407" daemon prio=5 tid=201 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"Time-limited test" daemon prio=5 tid=22 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1864)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1898)
        at java.base@17.0.11/java.util.concurrent.CompletableFuture.join(CompletableFuture.java:2117)
        at app//org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi.clearPeerAndQueues(TestAsyncReplicationAdminApi.java:96)
        at java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
        at java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568)
        at app//org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
        at app//org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
        at app//org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
        at app//org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
        at app//org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
        at app//org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
        at app//org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
        at app//org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
        at app//org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
        at app//org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
        at app//org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
        at app//org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
        at app//org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
        at app//org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
        at app//org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
        at app//org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
        at app//org.junit.runners.ParentRunner.run(ParentRunner.java:413)
        at app//org.junit.runners.Suite.runChild(Suite.java:128)
        at app//org.junit.runners.Suite.runChild(Suite.java:27)
        at app//org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
        at app//org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
        at app//org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
        at app//org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
        at app//org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
        at app//org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
        at app//org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
        at app//org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
        at app//org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
        at java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"FSEditLogAsync" daemon prio=5 tid=53 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
        at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:241)
        at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:250)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server handler 0 on default port 42041" daemon prio=5 tid=64 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"BP-956810084-172.17.0.2-1731118830705 heartbeating to localhost/127.0.0.1:42041" daemon prio=5 tid=102 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:771)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:914)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RS-EventLoopGroup-3-3" daemon prio=10 tid=293 runnable
java.lang.Thread.State: RUNNABLE
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Command processor" daemon prio=5 tid=101 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395)
"qtp1312628392-43" daemon prio=5 tid=43 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@7a30b880" daemon prio=5 tid=73 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:4592)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"pool-6-thread-1"  prio=5 tid=46 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Parameter Sending Thread for localhost/127.0.0.1:42041" daemon prio=5 tid=110 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
        at java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-16" daemon prio=5 tid=172 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"region-location-0" daemon prio=5 tid=317 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40627" daemon prio=5 tid=178 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"FsDatasetAsyncDiskServiceFixer" daemon prio=5 tid=136 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer.run(HBaseTestingUtility.java:599)
"RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37407" daemon prio=5 tid=199 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100" daemon prio=5 tid=35 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)
"IPC Server Responder" daemon prio=5 tid=57 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733)
        at app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716)
"RS:0;54f3333e9615:37407" daemon prio=5 tid=206 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:81)
        at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:64)
        at app//org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1164)
        at app//org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(MiniHBaseCluster.java:176)
        at app//org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.access$000(MiniHBaseCluster.java:124)
        at app//org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer$1.run(MiniHBaseCluster.java:160)
        at java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776)
        at java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:399)
        at java.base@17.0.11/javax.security.auth.Subject.doAs(Subject.java:376)
        at app//org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
        at app//org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
        at app//org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.run(MiniHBaseCluster.java:157)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"PEWorker-4" daemon prio=5 tid=248 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149)
        at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2128)
"RegionServerTracker-0" daemon prio=5 tid=268 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"regionserver/54f3333e9615:0.logRoller" daemon prio=5 tid=275 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:179)
"IPC Server handler 1 on default port 42041" daemon prio=5 tid=65 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@a467a69" daemon prio=5 tid=85 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method)
        at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
        at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
        at java.base@17.0.11/sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:126)
        at app//org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:242)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=37407" daemon prio=5 tid=203 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"DataXceiver for client DFSClient_NONMAPREDUCE_-1728371911_22 at /127.0.0.1:44078 [Receiving block BP-956810084-172.17.0.2-1731118830705:blk_1073741834_1010]" daemon prio=5 tid=295 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
        at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
        at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
        at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
        at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
        at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151)
        at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"surefire-forkedjvm-stream-flusher" daemon prio=5 tid=16 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-9" daemon prio=5 tid=165 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"master/54f3333e9615:0:becomeActiveMaster-HFileCleaner.large.0-1731118836768" daemon prio=5 tid=265 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at app//org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:101)
        at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner.consumerLoop(HFileCleaner.java:285)
        at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner$1.run(HFileCleaner.java:254)
"RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37407" daemon prio=5 tid=195 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"M:0;54f3333e9615:40627" daemon prio=5 tid=184 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:81)
        at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:64)
        at app//org.apache.hadoop.hbase.master.HMaster.waitForMasterActive(HMaster.java:702)
        at app//org.apache.hadoop.hbase.regionserver.HRegionServer.initializeZooKeeper(HRegionServer.java:1014)
        at app//org.apache.hadoop.hbase.regionserver.HRegionServer.preRegistrationInitialization(HRegionServer.java:954)
        at app//org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1075)
        at app//org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:603)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Time-limited test-EventThread" daemon prio=5 tid=404 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
"qtp1312628392-44" daemon prio=5 tid=44 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Time-limited test-SendThread(127.0.0.1:60825)" daemon prio=5 tid=155 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
"qtp1965036093-88-acceptor-0@597ed123-ServerConnector@723ec49{HTTP/1.1, (http/1.1)}{localhost:43405}" daemon prio=3 tid=88 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method)
        at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
        at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ProcessThread(sid:0 cport:60825):" daemon prio=5 tid=143 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
"NIOWorkerThread-15" daemon prio=5 tid=171 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Time-limited test-SendThread(127.0.0.1:60825)" daemon prio=5 tid=192 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
"snapshot-hfile-cleaner-cache-refresher" daemon prio=5 tid=264 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
        at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
"Reference Handler" daemon prio=10 tid=2 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/java.lang.ref.Reference.waitForReferencePendingList(Native Method)
        at java.base@17.0.11/java.lang.ref.Reference.processPendingReferences(Reference.java:253)
        at java.base@17.0.11/java.lang.ref.Reference$ReferenceHandler.run(Reference.java:215)
"Time-limited test.LruBlockCache.EvictionThread" daemon prio=5 tid=187 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:957)
"IPC Server handler 2 on default port 46271" daemon prio=5 tid=106 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"PEWorker-3" daemon prio=5 tid=247 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149)
        at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2128)
"DataXceiver for client DFSClient_NONMAPREDUCE_-1157186250_22 at /127.0.0.1:39638 [Receiving block BP-956810084-172.17.0.2-1731118830705:blk_1073741840_1016]" daemon prio=5 tid=792 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
        at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
        at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
        at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
        at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
        at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151)
        at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=40627" daemon prio=5 tid=180 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
"ProcedureDispatcherTimeoutThread" daemon prio=5 tid=250 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81)
        at app//org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher$TimeoutExecutorThread.run(RemoteProcedureDispatcher.java:328)
"IPC Server handler 4 on default port 46271" daemon prio=5 tid=108 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"IPC Server listener on 0" daemon prio=5 tid=54 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559)
"Socket Reader #1 for port 0" daemon prio=5 tid=55 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497)
        at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476)