2024-12-06 08:01:01,143 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec
2024-12-06 08:01:01,154 main DEBUG Took 0.009548 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-06 08:01:01,154 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-06 08:01:01,155 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-06 08:01:01,155 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-06 08:01:01,157 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 08:01:01,173 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-06 08:01:01,184 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,186 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 08:01:01,186 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,187 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 08:01:01,187 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,187 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 08:01:01,188 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,188 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 08:01:01,189 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,189 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 08:01:01,190 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,190 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 08:01:01,190 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,191 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 08:01:01,191 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,191 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 08:01:01,192 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,192 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 08:01:01,193 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,193 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 08:01:01,193 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,194 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 08:01:01,194 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,194 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 08:01:01,195 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,195 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-06 08:01:01,196 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 08:01:01,198 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-06 08:01:01,199 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-06 08:01:01,200 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-06 08:01:01,201 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-06 08:01:01,201 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-06 08:01:01,209 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-06 08:01:01,211 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-06 08:01:01,213 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-06 08:01:01,213 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-06 08:01:01,214 main DEBUG createAppenders(={Console})
2024-12-06 08:01:01,214 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec initialized
2024-12-06 08:01:01,215 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec
2024-12-06 08:01:01,215 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec OK.
2024-12-06 08:01:01,215 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-06 08:01:01,216 main DEBUG OutputStream closed
2024-12-06 08:01:01,216 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-06 08:01:01,216 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-06 08:01:01,216 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4efc180e OK
2024-12-06 08:01:01,301 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-06 08:01:01,303 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-06 08:01:01,304 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-06 08:01:01,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-06 08:01:01,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-06 08:01:01,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-06 08:01:01,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-06 08:01:01,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-06 08:01:01,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-06 08:01:01,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-06 08:01:01,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-06 08:01:01,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-06 08:01:01,308 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-06 08:01:01,308 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-06 08:01:01,308 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-06 08:01:01,309 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-06 08:01:01,309 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-06 08:01:01,310 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-06 08:01:01,312 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-06 08:01:01,313 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@39de3d36) with optional ClassLoader: null
2024-12-06 08:01:01,313 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-06 08:01:01,314 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@39de3d36] started OK.
2024-12-06T08:01:01,558 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe
2024-12-06 08:01:01,561 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-06 08:01:01,561 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-06T08:01:01,586 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi timeout: 26 mins
2024-12-06T08:01:01,603 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T08:01:01,617 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/cluster_acdf4e60-e0ee-bbdc-8ae2-647dd3ea8ddc, deleteOnExit=true
2024-12-06T08:01:01,617 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T08:01:01,618 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/test.cache.data in system properties and HBase conf
2024-12-06T08:01:01,619 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T08:01:01,619 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/hadoop.log.dir in system properties and HBase conf
2024-12-06T08:01:01,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T08:01:01,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T08:01:01,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T08:01:01,706 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-06T08:01:01,796 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T08:01:01,799 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T08:01:01,800 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T08:01:01,800 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T08:01:01,801 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T08:01:01,801 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T08:01:01,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T08:01:01,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T08:01:01,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T08:01:01,803 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T08:01:01,803 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/nfs.dump.dir in system properties and HBase conf
2024-12-06T08:01:01,804 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/java.io.tmpdir in system properties and HBase conf
2024-12-06T08:01:01,804 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T08:01:01,804 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T08:01:01,805 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T08:01:02,662 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-12-06T08:01:02,743 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2318ms to org.eclipse.jetty.util.log.Slf4jLog
2024-12-06T08:01:02,818 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T08:01:02,880 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T08:01:02,900 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T08:01:02,901 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T08:01:02,902 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T08:01:02,915 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T08:01:02,917 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33fb8453{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/hadoop.log.dir/,AVAILABLE}
2024-12-06T08:01:02,918 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b133317{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T08:01:03,137 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1441d181{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/java.io.tmpdir/jetty-localhost-34833-hadoop-hdfs-3_4_1-tests_jar-_-any-10865360016114309774/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T08:01:03,144 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@135deabe{HTTP/1.1, (http/1.1)}{localhost:34833}
2024-12-06T08:01:03,145 INFO  [Time-limited test {}] server.Server(415): Started @2720ms
2024-12-06T08:01:03,534 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T08:01:03,541 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T08:01:03,542 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T08:01:03,542 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T08:01:03,543 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T08:01:03,543 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@698dfd8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/hadoop.log.dir/,AVAILABLE}
2024-12-06T08:01:03,544 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62b38b68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T08:01:03,666 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2e80f2cb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/java.io.tmpdir/jetty-localhost-40603-hadoop-hdfs-3_4_1-tests_jar-_-any-4143313603187906521/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T08:01:03,667 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43176244{HTTP/1.1, (http/1.1)}{localhost:40603}
2024-12-06T08:01:03,667 INFO  [Time-limited test {}] server.Server(415): Started @3242ms
2024-12-06T08:01:03,722 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T08:01:04,171 WARN  [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/cluster_acdf4e60-e0ee-bbdc-8ae2-647dd3ea8ddc/data/data2/current/BP-367880231-172.17.0.2-1733472062403/current, will proceed with Du for space computation calculation, 
2024-12-06T08:01:04,171 WARN  [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/cluster_acdf4e60-e0ee-bbdc-8ae2-647dd3ea8ddc/data/data1/current/BP-367880231-172.17.0.2-1733472062403/current, will proceed with Du for space computation calculation, 
2024-12-06T08:01:04,209 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T08:01:04,269 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe
2024-12-06T08:01:04,270 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x444fc7d0ebdd4b94 with lease ID 0x3f5a25b9bfbf27cf: Processing first storage report for DS-2734aae5-af90-40e7-b9d8-6e9731ae49a4 from datanode DatanodeRegistration(127.0.0.1:32829, datanodeUuid=36de512a-ffaf-4ab4-82b2-0427682b3dff, infoPort=39251, infoSecurePort=0, ipcPort=38083, storageInfo=lv=-57;cid=testClusterID;nsid=1439152794;c=1733472062403)
2024-12-06T08:01:04,271 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x444fc7d0ebdd4b94 with lease ID 0x3f5a25b9bfbf27cf: from storage DS-2734aae5-af90-40e7-b9d8-6e9731ae49a4 node DatanodeRegistration(127.0.0.1:32829, datanodeUuid=36de512a-ffaf-4ab4-82b2-0427682b3dff, infoPort=39251, infoSecurePort=0, ipcPort=38083, storageInfo=lv=-57;cid=testClusterID;nsid=1439152794;c=1733472062403), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T08:01:04,271 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x444fc7d0ebdd4b94 with lease ID 0x3f5a25b9bfbf27cf: Processing first storage report for DS-1302926f-1e77-44e2-807b-0c6a986e85ff from datanode DatanodeRegistration(127.0.0.1:32829, datanodeUuid=36de512a-ffaf-4ab4-82b2-0427682b3dff, infoPort=39251, infoSecurePort=0, ipcPort=38083, storageInfo=lv=-57;cid=testClusterID;nsid=1439152794;c=1733472062403)
2024-12-06T08:01:04,271 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x444fc7d0ebdd4b94 with lease ID 0x3f5a25b9bfbf27cf: from storage DS-1302926f-1e77-44e2-807b-0c6a986e85ff node DatanodeRegistration(127.0.0.1:32829, datanodeUuid=36de512a-ffaf-4ab4-82b2-0427682b3dff, infoPort=39251, infoSecurePort=0, ipcPort=38083, storageInfo=lv=-57;cid=testClusterID;nsid=1439152794;c=1733472062403), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T08:01:04,352 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/cluster_acdf4e60-e0ee-bbdc-8ae2-647dd3ea8ddc/zookeeper_0, clientPort=64333, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/cluster_acdf4e60-e0ee-bbdc-8ae2-647dd3ea8ddc/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/cluster_acdf4e60-e0ee-bbdc-8ae2-647dd3ea8ddc/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T08:01:04,363 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64333
2024-12-06T08:01:04,376 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T08:01:04,379 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T08:01:04,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741825_1001 (size=7)
2024-12-06T08:01:05,012 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2 with version=8
2024-12-06T08:01:05,012 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/hbase-staging
2024-12-06T08:01:05,115 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-12-06T08:01:05,364 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/aa9080fe83d2:0 server-side Connection retries=6
2024-12-06T08:01:05,375 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T08:01:05,375 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T08:01:05,380 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T08:01:05,380 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T08:01:05,380 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T08:01:05,514 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T08:01:05,579 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-06T08:01:05,592 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-06T08:01:05,598 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T08:01:05,626 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 19831 (auto-detected)
2024-12-06T08:01:05,627 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-12-06T08:01:05,646 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39057
2024-12-06T08:01:05,667 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39057 connecting to ZooKeeper ensemble=127.0.0.1:64333
2024-12-06T08:01:05,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:390570x0, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T08:01:05,704 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39057-0x10186ca7dde0000 connected
2024-12-06T08:01:05,733 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T08:01:05,735 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T08:01:05,745 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T08:01:05,749 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2, hbase.cluster.distributed=false
2024-12-06T08:01:05,775 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T08:01:05,780 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39057
2024-12-06T08:01:05,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39057
2024-12-06T08:01:05,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39057
2024-12-06T08:01:05,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39057
2024-12-06T08:01:05,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39057
2024-12-06T08:01:05,892 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/aa9080fe83d2:0 server-side Connection retries=6
2024-12-06T08:01:05,894 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T08:01:05,894 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T08:01:05,894 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T08:01:05,895 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T08:01:05,895 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T08:01:05,897 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T08:01:05,900 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T08:01:05,900 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46375
2024-12-06T08:01:05,902 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46375 connecting to ZooKeeper ensemble=127.0.0.1:64333
2024-12-06T08:01:05,904 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T08:01:05,908 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T08:01:05,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:463750x0, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T08:01:05,916 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46375-0x10186ca7dde0001 connected
2024-12-06T08:01:05,916 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T08:01:05,921 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T08:01:05,929 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T08:01:05,931 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T08:01:05,935 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T08:01:05,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46375
2024-12-06T08:01:05,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46375
2024-12-06T08:01:05,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46375
2024-12-06T08:01:05,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46375
2024-12-06T08:01:05,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46375
2024-12-06T08:01:05,953 DEBUG [M:0;aa9080fe83d2:39057 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;aa9080fe83d2:39057
2024-12-06T08:01:05,954 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/aa9080fe83d2,39057,1733472065171
2024-12-06T08:01:05,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T08:01:05,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T08:01:05,962 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/aa9080fe83d2,39057,1733472065171
2024-12-06T08:01:05,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T08:01:05,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:05,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:05,986 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T08:01:05,987 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/aa9080fe83d2,39057,1733472065171 from backup master directory
2024-12-06T08:01:05,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/aa9080fe83d2,39057,1733472065171
2024-12-06T08:01:05,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T08:01:05,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T08:01:05,990 WARN  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T08:01:05,990 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=aa9080fe83d2,39057,1733472065171
2024-12-06T08:01:05,992 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-12-06T08:01:05,993 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-12-06T08:01:06,052 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/hbase.id] with ID: c9b453b4-ba9a-4f5f-80a1-c9cf6cbc0ebc
2024-12-06T08:01:06,052 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/.tmp/hbase.id
2024-12-06T08:01:06,062 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741826_1002 (size=42)
2024-12-06T08:01:06,464 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/.tmp/hbase.id]:[hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/hbase.id]
2024-12-06T08:01:06,507 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T08:01:06,512 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T08:01:06,532 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-12-06T08:01:06,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:06,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:06,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741827_1003 (size=196)
2024-12-06T08:01:06,967 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T08:01:06,971 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T08:01:06,994 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396
java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo)
	at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.<clinit>(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:262) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:231) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:06,999 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-06T08:01:07,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741828_1004 (size=1189)
2024-12-06T08:01:07,451 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store
2024-12-06T08:01:07,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741829_1005 (size=34)
2024-12-06T08:01:07,476 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-12-06T08:01:07,479 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T08:01:07,480 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T08:01:07,480 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T08:01:07,480 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T08:01:07,482 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T08:01:07,482 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T08:01:07,482 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T08:01:07,483 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733472067480Disabling compacts and flushes for region at 1733472067480Disabling writes for close at 1733472067482 (+2 ms)Writing region close event to WAL at 1733472067482Closed at 1733472067482
2024-12-06T08:01:07,485 WARN  [master/aa9080fe83d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/.initializing
2024-12-06T08:01:07,485 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/WALs/aa9080fe83d2,39057,1733472065171
2024-12-06T08:01:07,493 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-12-06T08:01:07,507 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aa9080fe83d2%2C39057%2C1733472065171, suffix=, logDir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/WALs/aa9080fe83d2,39057,1733472065171, archiveDir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/oldWALs, maxLogs=10
2024-12-06T08:01:07,530 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/WALs/aa9080fe83d2,39057,1733472065171/aa9080fe83d2%2C39057%2C1733472065171.1733472067512, exclude list is [], retry=0
2024-12-06T08:01:07,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32829,DS-2734aae5-af90-40e7-b9d8-6e9731ae49a4,DISK]
2024-12-06T08:01:07,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf.
2024-12-06T08:01:07,585 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/WALs/aa9080fe83d2,39057,1733472065171/aa9080fe83d2%2C39057%2C1733472065171.1733472067512
2024-12-06T08:01:07,586 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39251:39251)]
2024-12-06T08:01:07,586 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T08:01:07,587 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T08:01:07,590 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T08:01:07,591 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T08:01:07,628 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T08:01:07,655 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T08:01:07,659 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:07,661 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T08:01:07,661 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T08:01:07,664 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T08:01:07,665 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:07,666 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T08:01:07,666 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T08:01:07,668 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T08:01:07,669 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:07,670 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T08:01:07,670 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T08:01:07,672 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T08:01:07,673 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:07,674 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T08:01:07,674 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T08:01:07,677 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T08:01:07,678 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T08:01:07,683 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T08:01:07,684 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T08:01:07,688 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T08:01:07,692 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T08:01:07,696 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T08:01:07,697 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69573951, jitterRate=0.03673265874385834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T08:01:07,706 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733472067603Initializing all the Stores at 1733472067605 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733472067606 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733472067606Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733472067607 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733472067607Cleaning up temporary data from old regions at 1733472067684 (+77 ms)Region opened successfully at 1733472067706 (+22 ms)
2024-12-06T08:01:07,707 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T08:01:07,748 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@496b3bc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aa9080fe83d2/172.17.0.2:0
2024-12-06T08:01:07,780 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T08:01:07,791 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T08:01:07,791 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T08:01:07,794 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T08:01:07,795 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-12-06T08:01:07,801 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-12-06T08:01:07,801 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T08:01:07,826 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T08:01:07,834 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T08:01:07,836 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T08:01:07,839 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T08:01:07,840 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T08:01:07,843 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T08:01:07,845 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T08:01:07,849 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T08:01:07,851 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T08:01:07,852 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T08:01:07,854 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T08:01:07,872 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T08:01:07,873 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T08:01:07,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T08:01:07,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T08:01:07,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:07,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:07,882 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=aa9080fe83d2,39057,1733472065171, sessionid=0x10186ca7dde0000, setting cluster-up flag (Was=false)
2024-12-06T08:01:07,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:07,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:07,904 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T08:01:07,906 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=aa9080fe83d2,39057,1733472065171
2024-12-06T08:01:07,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:07,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:07,918 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T08:01:07,920 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=aa9080fe83d2,39057,1733472065171
2024-12-06T08:01:07,926 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T08:01:07,943 INFO  [RS:0;aa9080fe83d2:46375 {}] regionserver.HRegionServer(746): ClusterId : c9b453b4-ba9a-4f5f-80a1-c9cf6cbc0ebc
2024-12-06T08:01:07,946 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T08:01:07,951 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T08:01:07,951 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T08:01:07,958 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T08:01:07,959 DEBUG [RS:0;aa9080fe83d2:46375 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4020c6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aa9080fe83d2/172.17.0.2:0
2024-12-06T08:01:07,978 DEBUG [RS:0;aa9080fe83d2:46375 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;aa9080fe83d2:46375
2024-12-06T08:01:07,983 INFO  [RS:0;aa9080fe83d2:46375 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T08:01:07,983 INFO  [RS:0;aa9080fe83d2:46375 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T08:01:07,983 DEBUG [RS:0;aa9080fe83d2:46375 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T08:01:07,986 INFO  [RS:0;aa9080fe83d2:46375 {}] regionserver.HRegionServer(2659): reportForDuty to master=aa9080fe83d2,39057,1733472065171 with port=46375, startcode=1733472065852
2024-12-06T08:01:07,998 DEBUG [RS:0;aa9080fe83d2:46375 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T08:01:08,003 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T08:01:08,015 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T08:01:08,024 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T08:01:08,031 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: aa9080fe83d2,39057,1733472065171
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T08:01:08,037 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34613, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T08:01:08,039 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/aa9080fe83d2:0, corePoolSize=5, maxPoolSize=5
2024-12-06T08:01:08,039 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/aa9080fe83d2:0, corePoolSize=5, maxPoolSize=5
2024-12-06T08:01:08,039 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/aa9080fe83d2:0, corePoolSize=5, maxPoolSize=5
2024-12-06T08:01:08,039 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/aa9080fe83d2:0, corePoolSize=5, maxPoolSize=5
2024-12-06T08:01:08,040 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/aa9080fe83d2:0, corePoolSize=10, maxPoolSize=10
2024-12-06T08:01:08,040 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/aa9080fe83d2:0, corePoolSize=1, maxPoolSize=1
2024-12-06T08:01:08,040 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/aa9080fe83d2:0, corePoolSize=2, maxPoolSize=2
2024-12-06T08:01:08,041 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/aa9080fe83d2:0, corePoolSize=1, maxPoolSize=1
2024-12-06T08:01:08,045 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733472098045
2024-12-06T08:01:08,045 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39057 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
	at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:08,047 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T08:01:08,047 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T08:01:08,047 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T08:01:08,048 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T08:01:08,052 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T08:01:08,052 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T08:01:08,053 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T08:01:08,053 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T08:01:08,056 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:08,056 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T08:01:08,055 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,063 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T08:01:08,065 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T08:01:08,065 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T08:01:08,073 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T08:01:08,074 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T08:01:08,076 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/aa9080fe83d2:0:becomeActiveMaster-HFileCleaner.large.0-1733472068075,5,FailOnTimeoutGroup]
2024-12-06T08:01:08,076 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741831_1007 (size=1321)
2024-12-06T08:01:08,078 DEBUG [RS:0;aa9080fe83d2:46375 {}] regionserver.HRegionServer(2683): Master is not running yet
2024-12-06T08:01:08,078 WARN  [RS:0;aa9080fe83d2:46375 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying.
2024-12-06T08:01:08,079 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T08:01:08,079 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2
2024-12-06T08:01:08,080 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/aa9080fe83d2:0:becomeActiveMaster-HFileCleaner.small.0-1733472068076,5,FailOnTimeoutGroup]
2024-12-06T08:01:08,080 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,081 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T08:01:08,082 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,082 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741832_1008 (size=32)
2024-12-06T08:01:08,180 INFO  [RS:0;aa9080fe83d2:46375 {}] regionserver.HRegionServer(2659): reportForDuty to master=aa9080fe83d2,39057,1733472065171 with port=46375, startcode=1733472065852
2024-12-06T08:01:08,182 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39057 {}] master.ServerManager(363): Checking decommissioned status of RegionServer aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:08,185 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39057 {}] master.ServerManager(517): Registering regionserver=aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:08,193 DEBUG [RS:0;aa9080fe83d2:46375 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2
2024-12-06T08:01:08,193 DEBUG [RS:0;aa9080fe83d2:46375 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34433
2024-12-06T08:01:08,193 DEBUG [RS:0;aa9080fe83d2:46375 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T08:01:08,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T08:01:08,198 DEBUG [RS:0;aa9080fe83d2:46375 {}] zookeeper.ZKUtil(111): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:08,199 WARN  [RS:0;aa9080fe83d2:46375 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T08:01:08,199 INFO  [RS:0;aa9080fe83d2:46375 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-06T08:01:08,199 DEBUG [RS:0;aa9080fe83d2:46375 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/WALs/aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:08,200 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [aa9080fe83d2,46375,1733472065852]
2024-12-06T08:01:08,228 INFO  [RS:0;aa9080fe83d2:46375 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T08:01:08,242 INFO  [RS:0;aa9080fe83d2:46375 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T08:01:08,248 INFO  [RS:0;aa9080fe83d2:46375 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T08:01:08,248 INFO  [RS:0;aa9080fe83d2:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,249 INFO  [RS:0;aa9080fe83d2:46375 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T08:01:08,256 INFO  [RS:0;aa9080fe83d2:46375 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T08:01:08,258 INFO  [RS:0;aa9080fe83d2:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,259 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/aa9080fe83d2:0, corePoolSize=1, maxPoolSize=1
2024-12-06T08:01:08,259 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/aa9080fe83d2:0, corePoolSize=1, maxPoolSize=1
2024-12-06T08:01:08,259 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0, corePoolSize=1, maxPoolSize=1
2024-12-06T08:01:08,259 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/aa9080fe83d2:0, corePoolSize=1, maxPoolSize=1
2024-12-06T08:01:08,259 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/aa9080fe83d2:0, corePoolSize=1, maxPoolSize=1
2024-12-06T08:01:08,260 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/aa9080fe83d2:0, corePoolSize=2, maxPoolSize=2
2024-12-06T08:01:08,260 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/aa9080fe83d2:0, corePoolSize=1, maxPoolSize=1
2024-12-06T08:01:08,260 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/aa9080fe83d2:0, corePoolSize=1, maxPoolSize=1
2024-12-06T08:01:08,260 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/aa9080fe83d2:0, corePoolSize=1, maxPoolSize=1
2024-12-06T08:01:08,261 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/aa9080fe83d2:0, corePoolSize=1, maxPoolSize=1
2024-12-06T08:01:08,261 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/aa9080fe83d2:0, corePoolSize=1, maxPoolSize=1
2024-12-06T08:01:08,261 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/aa9080fe83d2:0, corePoolSize=1, maxPoolSize=1
2024-12-06T08:01:08,261 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/aa9080fe83d2:0, corePoolSize=3, maxPoolSize=3
2024-12-06T08:01:08,262 DEBUG [RS:0;aa9080fe83d2:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/aa9080fe83d2:0, corePoolSize=3, maxPoolSize=3
2024-12-06T08:01:08,263 INFO  [RS:0;aa9080fe83d2:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,264 INFO  [RS:0;aa9080fe83d2:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,264 INFO  [RS:0;aa9080fe83d2:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,264 INFO  [RS:0;aa9080fe83d2:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,264 INFO  [RS:0;aa9080fe83d2:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,264 INFO  [RS:0;aa9080fe83d2:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=aa9080fe83d2,46375,1733472065852-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T08:01:08,288 INFO  [RS:0;aa9080fe83d2:46375 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T08:01:08,290 INFO  [RS:0;aa9080fe83d2:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=aa9080fe83d2,46375,1733472065852-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,290 INFO  [RS:0;aa9080fe83d2:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,290 INFO  [RS:0;aa9080fe83d2:46375 {}] regionserver.Replication(171): aa9080fe83d2,46375,1733472065852 started
2024-12-06T08:01:08,308 INFO  [RS:0;aa9080fe83d2:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:08,309 INFO  [RS:0;aa9080fe83d2:46375 {}] regionserver.HRegionServer(1482): Serving as aa9080fe83d2,46375,1733472065852, RpcServer on aa9080fe83d2/172.17.0.2:46375, sessionid=0x10186ca7dde0001
2024-12-06T08:01:08,310 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T08:01:08,310 DEBUG [RS:0;aa9080fe83d2:46375 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:08,310 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aa9080fe83d2,46375,1733472065852'
2024-12-06T08:01:08,310 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T08:01:08,311 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T08:01:08,312 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T08:01:08,312 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T08:01:08,312 DEBUG [RS:0;aa9080fe83d2:46375 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:08,312 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aa9080fe83d2,46375,1733472065852'
2024-12-06T08:01:08,312 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T08:01:08,313 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T08:01:08,314 DEBUG [RS:0;aa9080fe83d2:46375 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T08:01:08,314 INFO  [RS:0;aa9080fe83d2:46375 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T08:01:08,314 INFO  [RS:0;aa9080fe83d2:46375 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T08:01:08,419 INFO  [RS:0;aa9080fe83d2:46375 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-12-06T08:01:08,422 INFO  [RS:0;aa9080fe83d2:46375 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aa9080fe83d2%2C46375%2C1733472065852, suffix=, logDir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/WALs/aa9080fe83d2,46375,1733472065852, archiveDir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/oldWALs, maxLogs=32
2024-12-06T08:01:08,439 DEBUG [RS:0;aa9080fe83d2:46375 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/WALs/aa9080fe83d2,46375,1733472065852/aa9080fe83d2%2C46375%2C1733472065852.1733472068425, exclude list is [], retry=0
2024-12-06T08:01:08,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32829,DS-2734aae5-af90-40e7-b9d8-6e9731ae49a4,DISK]
2024-12-06T08:01:08,447 INFO  [RS:0;aa9080fe83d2:46375 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/WALs/aa9080fe83d2,46375,1733472065852/aa9080fe83d2%2C46375%2C1733472065852.1733472068425
2024-12-06T08:01:08,447 DEBUG [RS:0;aa9080fe83d2:46375 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39251:39251)]
2024-12-06T08:01:08,494 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T08:01:08,497 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T08:01:08,500 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T08:01:08,500 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:08,501 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T08:01:08,501 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T08:01:08,503 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T08:01:08,503 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:08,504 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T08:01:08,504 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T08:01:08,506 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T08:01:08,506 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:08,507 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T08:01:08,508 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T08:01:08,510 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T08:01:08,510 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:08,510 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T08:01:08,511 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T08:01:08,512 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740
2024-12-06T08:01:08,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740
2024-12-06T08:01:08,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T08:01:08,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T08:01:08,516 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T08:01:08,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T08:01:08,522 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T08:01:08,523 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73051989, jitterRate=0.08855946362018585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T08:01:08,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733472068495Initializing all the Stores at 1733472068496 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733472068497 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733472068497Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733472068497Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733472068497Cleaning up temporary data from old regions at 1733472068516 (+19 ms)Region opened successfully at 1733472068526 (+10 ms)
2024-12-06T08:01:08,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T08:01:08,526 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T08:01:08,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T08:01:08,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T08:01:08,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T08:01:08,528 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T08:01:08,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733472068526Disabling compacts and flushes for region at 1733472068526Disabling writes for close at 1733472068526Writing region close event to WAL at 1733472068527 (+1 ms)Closed at 1733472068527
2024-12-06T08:01:08,531 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T08:01:08,531 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T08:01:08,538 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T08:01:08,546 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T08:01:08,548 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T08:01:08,701 DEBUG [aa9080fe83d2:39057 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T08:01:08,713 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:08,720 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as aa9080fe83d2,46375,1733472065852, state=OPENING
2024-12-06T08:01:08,725 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T08:01:08,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:08,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:08,728 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T08:01:08,728 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T08:01:08,729 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T08:01:08,732 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=aa9080fe83d2,46375,1733472065852}]
2024-12-06T08:01:08,906 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T08:01:08,909 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40239, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T08:01:08,921 INFO  [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T08:01:08,921 INFO  [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-06T08:01:08,922 INFO  [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta
2024-12-06T08:01:08,926 INFO  [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aa9080fe83d2%2C46375%2C1733472065852.meta, suffix=.meta, logDir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/WALs/aa9080fe83d2,46375,1733472065852, archiveDir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/oldWALs, maxLogs=32
2024-12-06T08:01:08,944 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/WALs/aa9080fe83d2,46375,1733472065852/aa9080fe83d2%2C46375%2C1733472065852.meta.1733472068928.meta, exclude list is [], retry=0
2024-12-06T08:01:08,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32829,DS-2734aae5-af90-40e7-b9d8-6e9731ae49a4,DISK]
2024-12-06T08:01:08,951 INFO  [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/WALs/aa9080fe83d2,46375,1733472065852/aa9080fe83d2%2C46375%2C1733472065852.meta.1733472068928.meta
2024-12-06T08:01:08,952 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39251:39251)]
2024-12-06T08:01:08,952 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T08:01:08,955 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T08:01:08,958 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T08:01:08,964 INFO  [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T08:01:08,970 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T08:01:08,971 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T08:01:08,971 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T08:01:08,971 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T08:01:08,975 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T08:01:08,976 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T08:01:08,976 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:08,977 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T08:01:08,978 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T08:01:08,979 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T08:01:08,979 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:08,980 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T08:01:08,981 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T08:01:08,982 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T08:01:08,982 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:08,983 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T08:01:08,983 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T08:01:08,985 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T08:01:08,985 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:08,986 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T08:01:08,986 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T08:01:08,988 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740
2024-12-06T08:01:08,990 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740
2024-12-06T08:01:08,992 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T08:01:08,993 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T08:01:08,993 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T08:01:08,996 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T08:01:08,998 INFO  [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59593600, jitterRate=-0.11198616027832031}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T08:01:08,998 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T08:01:09,000 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733472068972Writing region info on filesystem at 1733472068972Initializing all the Stores at 1733472068974 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733472068974Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733472068974Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733472068974Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733472068975 (+1 ms)Cleaning up temporary data from old regions at 1733472068993 (+18 ms)Running coprocessor post-open hooks at 1733472068998 (+5 ms)Region opened successfully at 1733472069000 (+2 ms)
2024-12-06T08:01:09,010 INFO  [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733472068897
2024-12-06T08:01:09,021 DEBUG [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T08:01:09,022 INFO  [RS_OPEN_META-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T08:01:09,024 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:09,026 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as aa9080fe83d2,46375,1733472065852, state=OPEN
2024-12-06T08:01:09,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T08:01:09,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T08:01:09,036 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T08:01:09,036 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T08:01:09,036 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:09,041 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T08:01:09,041 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=aa9080fe83d2,46375,1733472065852 in 305 msec
2024-12-06T08:01:09,048 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T08:01:09,048 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 506 msec
2024-12-06T08:01:09,050 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T08:01:09,050 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T08:01:09,071 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T08:01:09,072 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aa9080fe83d2,46375,1733472065852, seqNum=-1]
2024-12-06T08:01:09,095 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T08:01:09,098 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57017, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T08:01:09,117 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1600 sec
2024-12-06T08:01:09,118 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733472069118, completionTime=-1
2024-12-06T08:01:09,120 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T08:01:09,120 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T08:01:09,146 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T08:01:09,147 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733472129146
2024-12-06T08:01:09,147 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733472189147
2024-12-06T08:01:09,147 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec
2024-12-06T08:01:09,149 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aa9080fe83d2,39057,1733472065171-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:09,149 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aa9080fe83d2,39057,1733472065171-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:09,149 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aa9080fe83d2,39057,1733472065171-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:09,151 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-aa9080fe83d2:39057, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:09,151 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:09,152 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:09,157 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T08:01:09,178 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.188sec
2024-12-06T08:01:09,179 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T08:01:09,180 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T08:01:09,181 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T08:01:09,182 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T08:01:09,182 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T08:01:09,182 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aa9080fe83d2,39057,1733472065171-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T08:01:09,183 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aa9080fe83d2,39057,1733472065171-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T08:01:09,192 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T08:01:09,192 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T08:01:09,193 INFO  [master/aa9080fe83d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aa9080fe83d2,39057,1733472065171-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T08:01:09,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f5b8af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T08:01:09,255 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-12-06T08:01:09,255 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-12-06T08:01:09,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aa9080fe83d2,39057,-1 for getting cluster id
2024-12-06T08:01:09,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T08:01:09,268 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c9b453b4-ba9a-4f5f-80a1-c9cf6cbc0ebc'
2024-12-06T08:01:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T08:01:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c9b453b4-ba9a-4f5f-80a1-c9cf6cbc0ebc"

2024-12-06T08:01:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51995f5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T08:01:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aa9080fe83d2,39057,-1]
2024-12-06T08:01:09,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T08:01:09,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T08:01:09,279 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T08:01:09,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a1cbff3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T08:01:09,282 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T08:01:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aa9080fe83d2,46375,1733472065852, seqNum=-1]
2024-12-06T08:01:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T08:01:09,291 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T08:01:09,314 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=aa9080fe83d2,39057,1733472065171
2024-12-06T08:01:09,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e9027b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T08:01:09,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aa9080fe83d2,39057,-1 for getting cluster id
2024-12-06T08:01:09,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T08:01:09,319 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c9b453b4-ba9a-4f5f-80a1-c9cf6cbc0ebc'
2024-12-06T08:01:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T08:01:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c9b453b4-ba9a-4f5f-80a1-c9cf6cbc0ebc"

2024-12-06T08:01:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eef864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T08:01:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aa9080fe83d2,39057,-1]
2024-12-06T08:01:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T08:01:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T08:01:09,324 INFO  [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T08:01:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@100cf2ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T08:01:09,338 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=559, ProcessCount=11, AvailableMemoryMB=4339
2024-12-06T08:01:09,366 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-06T08:01:09,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is aa9080fe83d2,39057,1733472065171
2024-12-06T08:01:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@657ce0d2
2024-12-06T08:01:09,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-06T08:01:09,379 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-06T08:01:09,380 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T08:01:09,400 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:09,405 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x19e0efa9 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3385717f
2024-12-06T08:01:09,405 WARN  [PEWorker-3 {}] client.ZKConnectionRegistry(87): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry
2024-12-06T08:01:09,411 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x19e0efa9 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x19e0efa9 to 127.0.0.1:64333
2024-12-06T08:01:09,412 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:09,413 INFO  [PEWorker-3 {}] master.HMaster(2490): Client=null/null create 'hbase:replication', {TABLE_ATTRIBUTES => {coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.regionserver.region.split_restriction.delimiter' => '-', 'hbase.regionserver.region.split_restriction.type' => 'DelimitedKeyPrefix'}}}, {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T08:01:09,419 DEBUG [PEWorker-3 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:replication
2024-12-06T08:01:09,422 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_PRE_OPERATION
2024-12-06T08:01:09,422 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:09,424 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-06T08:01:09,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741835_1011 (size=1138)
2024-12-06T08:01:09,471 DEBUG [PEWorker-3 {}] procedure.ProcedureSyncWait(219): waitFor Creating table hbase:replication
2024-12-06T08:01:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T08:01:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T08:01:09,841 INFO  [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5f2f3593e74f6cff3d351f05bde952ea, NAME => 'hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:replication', {TABLE_ATTRIBUTES => {coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.regionserver.region.split_restriction.delimiter' => '-', 'hbase.regionserver.region.split_restriction.type' => 'DelimitedKeyPrefix', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2
2024-12-06T08:01:09,849 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741836_1012 (size=44)
2024-12-06T08:01:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T08:01:10,250 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T08:01:10,251 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1722): Closing 5f2f3593e74f6cff3d351f05bde952ea, disabling compactions & flushes
2024-12-06T08:01:10,251 INFO  [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea.
2024-12-06T08:01:10,251 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea.
2024-12-06T08:01:10,251 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea. after waiting 0 ms
2024-12-06T08:01:10,251 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea.
2024-12-06T08:01:10,251 INFO  [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1973): Closed hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea.
2024-12-06T08:01:10,251 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5f2f3593e74f6cff3d351f05bde952ea:
Waiting for close lock at 1733472070251Disabling compacts and flushes for region at 1733472070251Disabling writes for close at 1733472070251Writing region close event to WAL at 1733472070251Closed at 1733472070251
2024-12-06T08:01:10,253 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_ADD_TO_META
2024-12-06T08:01:10,258 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea.","families":{"info":[{"qualifier":"regioninfo","vlen":43,"tag":[],"timestamp":"1733472070253"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733472070253"}]},"ts":"1733472070253"}
2024-12-06T08:01:10,263 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-06T08:01:10,265 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-06T08:01:10,269 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:replication","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733472070265"}]},"ts":"1733472070265"}
2024-12-06T08:01:10,273 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:replication, state=ENABLING in hbase:meta
2024-12-06T08:01:10,275 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=5f2f3593e74f6cff3d351f05bde952ea, ASSIGN}]
2024-12-06T08:01:10,277 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=5f2f3593e74f6cff3d351f05bde952ea, ASSIGN
2024-12-06T08:01:10,279 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:replication, region=5f2f3593e74f6cff3d351f05bde952ea, ASSIGN; state=OFFLINE, location=aa9080fe83d2,46375,1733472065852; forceNewPlan=false, retain=false
2024-12-06T08:01:10,430 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=5f2f3593e74f6cff3d351f05bde952ea, regionState=OPENING, regionLocation=aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:10,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:replication, region=5f2f3593e74f6cff3d351f05bde952ea, ASSIGN because future has completed
2024-12-06T08:01:10,435 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=7, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f2f3593e74f6cff3d351f05bde952ea, server=aa9080fe83d2,46375,1733472065852}]
2024-12-06T08:01:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T08:01:10,596 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] handler.AssignRegionHandler(132): Open hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea.
2024-12-06T08:01:10,596 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-06T08:01:10,597 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] monitor.StreamSlowMonitor(122): New stream slow monitor rep
2024-12-06T08:01:10,599 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aa9080fe83d2%2C46375%2C1733472065852.rep, suffix=, logDir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/WALs/aa9080fe83d2,46375,1733472065852, archiveDir=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/oldWALs, maxLogs=32
2024-12-06T08:01:10,615 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/WALs/aa9080fe83d2,46375,1733472065852/aa9080fe83d2%2C46375%2C1733472065852.rep.1733472070601, exclude list is [], retry=0
2024-12-06T08:01:10,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32829,DS-2734aae5-af90-40e7-b9d8-6e9731ae49a4,DISK]
2024-12-06T08:01:10,621 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/WALs/aa9080fe83d2,46375,1733472065852/aa9080fe83d2%2C46375%2C1733472065852.rep.1733472070601
2024-12-06T08:01:10,622 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39251:39251)]
2024-12-06T08:01:10,622 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f2f3593e74f6cff3d351f05bde952ea, NAME => 'hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea.', STARTKEY => '', ENDKEY => ''}
2024-12-06T08:01:10,622 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T08:01:10,622 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea. service=MultiRowMutationService
2024-12-06T08:01:10,623 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:replication successfully.
2024-12-06T08:01:10,623 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table replication 5f2f3593e74f6cff3d351f05bde952ea
2024-12-06T08:01:10,623 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(898): Instantiated hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T08:01:10,623 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7794): checking encryption for 5f2f3593e74f6cff3d351f05bde952ea
2024-12-06T08:01:10,623 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7797): checking classloading for 5f2f3593e74f6cff3d351f05bde952ea
2024-12-06T08:01:10,625 INFO  [StoreOpener-5f2f3593e74f6cff3d351f05bde952ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family hfileref of region 5f2f3593e74f6cff3d351f05bde952ea 
2024-12-06T08:01:10,628 INFO  [StoreOpener-5f2f3593e74f6cff3d351f05bde952ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f2f3593e74f6cff3d351f05bde952ea columnFamilyName hfileref
2024-12-06T08:01:10,628 DEBUG [StoreOpener-5f2f3593e74f6cff3d351f05bde952ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:10,630 INFO  [StoreOpener-5f2f3593e74f6cff3d351f05bde952ea-1 {}] regionserver.HStore(327): Store=5f2f3593e74f6cff3d351f05bde952ea/hfileref,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T08:01:10,630 INFO  [StoreOpener-5f2f3593e74f6cff3d351f05bde952ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family queue of region 5f2f3593e74f6cff3d351f05bde952ea 
2024-12-06T08:01:10,632 INFO  [StoreOpener-5f2f3593e74f6cff3d351f05bde952ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f2f3593e74f6cff3d351f05bde952ea columnFamilyName queue
2024-12-06T08:01:10,632 DEBUG [StoreOpener-5f2f3593e74f6cff3d351f05bde952ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:10,633 INFO  [StoreOpener-5f2f3593e74f6cff3d351f05bde952ea-1 {}] regionserver.HStore(327): Store=5f2f3593e74f6cff3d351f05bde952ea/queue,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T08:01:10,633 INFO  [StoreOpener-5f2f3593e74f6cff3d351f05bde952ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family sid of region 5f2f3593e74f6cff3d351f05bde952ea 
2024-12-06T08:01:10,636 INFO  [StoreOpener-5f2f3593e74f6cff3d351f05bde952ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f2f3593e74f6cff3d351f05bde952ea columnFamilyName sid
2024-12-06T08:01:10,636 DEBUG [StoreOpener-5f2f3593e74f6cff3d351f05bde952ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T08:01:10,637 INFO  [StoreOpener-5f2f3593e74f6cff3d351f05bde952ea-1 {}] regionserver.HStore(327): Store=5f2f3593e74f6cff3d351f05bde952ea/sid,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T08:01:10,637 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1038): replaying wal for 5f2f3593e74f6cff3d351f05bde952ea
2024-12-06T08:01:10,638 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/replication/5f2f3593e74f6cff3d351f05bde952ea
2024-12-06T08:01:10,639 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/replication/5f2f3593e74f6cff3d351f05bde952ea
2024-12-06T08:01:10,641 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1048): stopping wal replay for 5f2f3593e74f6cff3d351f05bde952ea
2024-12-06T08:01:10,641 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1060): Cleaning up temporary data for 5f2f3593e74f6cff3d351f05bde952ea
2024-12-06T08:01:10,642 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:replication descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead.
2024-12-06T08:01:10,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1093): writing seq id for 5f2f3593e74f6cff3d351f05bde952ea
2024-12-06T08:01:10,647 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/replication/5f2f3593e74f6cff3d351f05bde952ea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T08:01:10,648 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1114): Opened 5f2f3593e74f6cff3d351f05bde952ea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72549662, jitterRate=0.08107420802116394}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242}
2024-12-06T08:01:10,648 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f2f3593e74f6cff3d351f05bde952ea
2024-12-06T08:01:10,649 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1006): Region open journal for 5f2f3593e74f6cff3d351f05bde952ea:
Running coprocessor pre-open hook at 1733472070623Writing region info on filesystem at 1733472070623Initializing all the Stores at 1733472070625 (+2 ms)Instantiating store for column family {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733472070625Instantiating store for column family {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733472070625Instantiating store for column family {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733472070625Cleaning up temporary data from old regions at 1733472070641 (+16 ms)Running coprocessor post-open hooks at 1733472070648 (+7 ms)Region opened successfully at 1733472070649 (+1 ms)
2024-12-06T08:01:10,651 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea., pid=7, masterSystemTime=1733472070589
2024-12-06T08:01:10,654 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea.
2024-12-06T08:01:10,655 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] handler.AssignRegionHandler(153): Opened hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea.
2024-12-06T08:01:10,656 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=5f2f3593e74f6cff3d351f05bde952ea, regionState=OPEN, openSeqNum=2, regionLocation=aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:10,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f2f3593e74f6cff3d351f05bde952ea, server=aa9080fe83d2,46375,1733472065852 because future has completed
2024-12-06T08:01:10,666 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=7, resume processing ppid=6
2024-12-06T08:01:10,667 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f2f3593e74f6cff3d351f05bde952ea, server=aa9080fe83d2,46375,1733472065852 in 226 msec
2024-12-06T08:01:10,669 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-06T08:01:10,670 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=5f2f3593e74f6cff3d351f05bde952ea, ASSIGN in 391 msec
2024-12-06T08:01:10,671 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-06T08:01:10,672 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:replication","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733472070671"}]},"ts":"1733472070671"}
2024-12-06T08:01:10,675 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:replication, state=ENABLED in hbase:meta
2024-12-06T08:01:10,677 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_POST_OPERATION
2024-12-06T08:01:10,681 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:replication in 1.2630 sec
2024-12-06T08:01:10,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='1-', locateType=CURRENT is [region=hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea., hostname=aa9080fe83d2,46375,1733472065852, seqNum=2]
2024-12-06T08:01:10,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms
2024-12-06T08:01:10,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10186ca7dde0001, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:10,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T08:01:10,761 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=4, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:10,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=8
2024-12-06T08:01:10,926 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-12-06T08:01:10,968 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:10,969 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=8}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T08:01:10,969 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=8}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aa9080fe83d2,46375,1733472065852, seqNum=-1]
2024-12-06T08:01:10,970 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=8}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T08:01:10,972 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48041, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService
2024-12-06T08:01:10,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='1-aa9080fe83d2,46375,1733472065852', locateType=CURRENT is [region=hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea., hostname=aa9080fe83d2,46375,1733472065852, seqNum=2]
2024-12-06T08:01:10,983 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-12-06T08:01:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-12-06T08:01:10,990 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:10,995 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=4
2024-12-06T08:01:10,995 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 231 msec
2024-12-06T08:01:10,995 INFO  [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-12-06T08:01:10,999 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 1.6130 sec
2024-12-06T08:01:11,002 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:11,004 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x401ee957 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@39e9dff8
2024-12-06T08:01:11,008 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:11,008 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T08:01:11,547 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:11,550 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-06T08:01:11,557 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:11,559 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5dd7fb64 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ad78312
2024-12-06T08:01:11,564 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x5dd7fb64 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5dd7fb64 to 127.0.0.1:64333
2024-12-06T08:01:11,564 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:11,569 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:11,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-06T08:01:11,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=10
2024-12-06T08:01:11,723 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:11,725 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:11,725 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:11,726 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:11,726 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:11,728 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x66015df2 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6c5819ee
2024-12-06T08:01:11,728 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:11,747 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:11,747 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:11,747 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-12-06T08:01:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-12-06T08:01:11,752 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:11,756 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-12-06T08:01:11,756 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec
2024-12-06T08:01:11,756 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:11,759 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 206 msec
2024-12-06T08:01:11,762 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:11,764 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1779283c to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@74b4c577
2024-12-06T08:01:11,770 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:11,770 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-06T08:01:11,877 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:11,880 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:11,882 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:11,883 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:11,886 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false
2024-12-06T08:01:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-06T08:01:11,890 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:11,891 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x13c58b1a to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5d06a600
2024-12-06T08:01:11,895 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x13c58b1a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x13c58b1a to 127.0.0.1:64333
2024-12-06T08:01:11,895 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:11,900 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-06T08:01:12,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=12
2024-12-06T08:01:12,054 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:12,055 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:12,055 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:12,055 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:12,056 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:12,058 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x212a817d to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@67e10bb4
2024-12-06T08:01:12,058 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:12,078 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:12,078 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:12,078 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-12-06T08:01:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-12-06T08:01:12,082 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:12,086 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-12-06T08:01:12,086 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec
2024-12-06T08:01:12,086 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false
2024-12-06T08:01:12,088 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 200 msec
2024-12-06T08:01:12,095 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:12,096 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6696e4e2 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c9edf13
2024-12-06T08:01:12,099 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:12,100 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-06T08:01:12,206 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:12,207 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:12,209 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:12,210 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:12,211 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:12,213 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false
2024-12-06T08:01:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-06T08:01:12,218 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:12,219 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5c171776 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@16a8883
2024-12-06T08:01:12,223 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x5c171776 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5c171776 to 127.0.0.1:64333
2024-12-06T08:01:12,223 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:12,230 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-06T08:01:12,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=14
2024-12-06T08:01:12,384 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:12,386 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:12,386 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:12,386 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:12,386 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:12,388 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x56b58936 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@48a60731
2024-12-06T08:01:12,388 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:12,407 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:12,407 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:12,407 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-12-06T08:01:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-12-06T08:01:12,411 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:12,415 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-12-06T08:01:12,415 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 181 msec
2024-12-06T08:01:12,415 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false
2024-12-06T08:01:12,417 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 202 msec
2024-12-06T08:01:12,423 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:12,424 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0bd2d84c to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@75fd22ab
2024-12-06T08:01:12,428 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:12,428 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-06T08:01:12,536 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:12,537 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:12,538 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:12,539 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:12,541 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15
2024-12-06T08:01:12,545 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:12,548 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ae1eb0c to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f90eb6b
2024-12-06T08:01:12,551 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x7ae1eb0c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ae1eb0c to 127.0.0.1:64333
2024-12-06T08:01:12,551 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:12,560 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15
2024-12-06T08:01:12,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=16
2024-12-06T08:01:12,715 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:12,717 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:12,717 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:12,717 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:12,717 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:12,719 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1381c835 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3d9ec428
2024-12-06T08:01:12,720 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:12,739 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:12,739 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:12,740 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16
2024-12-06T08:01:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=16
2024-12-06T08:01:12,745 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:12,750 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15
2024-12-06T08:01:12,750 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 186 msec
2024-12-06T08:01:12,750 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:12,753 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 209 msec
2024-12-06T08:01:12,759 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:12,761 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x03800144 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7c2dbedc
2024-12-06T08:01:12,765 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:12,765 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15
2024-12-06T08:01:12,866 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:12,867 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:12,869 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:12,870 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false
2024-12-06T08:01:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17
2024-12-06T08:01:12,875 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:12,877 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x33b27621 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4ad12971
2024-12-06T08:01:12,882 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x33b27621 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x33b27621 to 127.0.0.1:64333
2024-12-06T08:01:12,882 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:12,887 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17
2024-12-06T08:01:13,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=18
2024-12-06T08:01:13,041 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:13,043 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:13,043 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:13,043 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:13,043 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:13,045 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ef65269 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6929bf8c
2024-12-06T08:01:13,045 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:13,072 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:13,072 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:13,073 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18
2024-12-06T08:01:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=18
2024-12-06T08:01:13,077 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:13,080 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17
2024-12-06T08:01:13,080 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 190 msec
2024-12-06T08:01:13,080 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false
2024-12-06T08:01:13,082 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 210 msec
2024-12-06T08:01:13,088 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:13,089 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x78c94197 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c07aa8d
2024-12-06T08:01:13,094 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:13,094 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17
2024-12-06T08:01:13,196 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:13,197 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:13,198 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19
2024-12-06T08:01:13,202 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:13,203 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x15fab522 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@70c69fe6
2024-12-06T08:01:13,206 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x15fab522 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x15fab522 to 127.0.0.1:64333
2024-12-06T08:01:13,207 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:13,211 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19
2024-12-06T08:01:13,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=20
2024-12-06T08:01:13,365 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:13,366 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:13,366 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:13,367 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:13,367 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:13,368 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2c7d4e25 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1ffa60f5
2024-12-06T08:01:13,369 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:13,386 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:13,386 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:13,387 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20
2024-12-06T08:01:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=20
2024-12-06T08:01:13,391 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:13,394 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19
2024-12-06T08:01:13,394 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:13,394 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec
2024-12-06T08:01:13,396 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 196 msec
2024-12-06T08:01:13,401 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:13,402 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x294e719b to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@ca14509
2024-12-06T08:01:13,405 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:13,406 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19
2024-12-06T08:01:13,517 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:13,517 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:13,521 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:13,521 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21
2024-12-06T08:01:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-12-06T08:01:13,529 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:13,530 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:13,530 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:13,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:13,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:13,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:13,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:13,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:13,536 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21
2024-12-06T08:01:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-12-06T08:01:13,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=23
2024-12-06T08:01:13,689 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-12-06T08:01:13,690 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-12-06T08:01:13,690 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Replication stream was removed by a user
2024-12-06T08:01:13,690 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:13,690 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:13,691 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63650f7f to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6d8d4fcf
2024-12-06T08:01:13,692 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:13,693 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSourceManager(562): Done with the queue 1-aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:13,697 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=23
2024-12-06T08:01:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=23
2024-12-06T08:01:13,700 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:13,703 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22
2024-12-06T08:01:13,703 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 164 msec
2024-12-06T08:01:13,703 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-12-06T08:01:13,704 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-12-06T08:01:13,733 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-12-06T08:01:13,736 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 212 msec
2024-12-06T08:01:13,737 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:13,745 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=21, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:13,746 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=21, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=224 msec
2024-12-06T08:01:13,746 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21
2024-12-06T08:01:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-12-06T08:01:13,847 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-12-06T08:01:13,847 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:13,848 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:13,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:13,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24
2024-12-06T08:01:13,851 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:13,853 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=24, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:13,854 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=24, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec
2024-12-06T08:01:13,854 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24
2024-12-06T08:01:13,957 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='', locateType=CURRENT is [region=hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea., hostname=aa9080fe83d2,46375,1733472065852, seqNum=2]
2024-12-06T08:01:13,992 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=266 (was 214)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x294e719b-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x78c94197-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1779283c
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x2c7d4e25-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_820960735_22 at /127.0.0.1:54240 [Receiving block BP-367880231-172.17.0.2-1733472062403:blk_1073741837_1013]
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
	app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
	java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
	java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
	java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
	java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151)
	app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119)
	app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553)
	app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011)
	app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110)
	app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x78c94197
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x56b58936-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x56b58936-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1ef65269
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_820960735_22 at /127.0.0.1:54132 [Waiting for operation #6]
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
	app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
	java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
	java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263)
	java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334)
	java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72)
	app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x66015df2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x2c7d4e25
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1381c835-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6696e4e2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6696e4e2-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x212a817d
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1381c835
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x212a817d-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x63650f7f-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x294e719b
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x401ee957-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool.commonPool-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x2c7d4e25-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1ef65269-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x0bd2d84c-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1381c835-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x294e719b-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x0bd2d84c-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x78c94197-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x0bd2d84c
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x66015df2-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2-prefix:aa9080fe83d2,46375,1733472065852.rep
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x212a817d-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1ef65269-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x03800144-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x401ee957-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x63650f7f-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1779283c-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x401ee957
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x03800144-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x63650f7f
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6696e4e2-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: RSProcedureDispatcher-pool-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RSProcedureDispatcher-pool-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: PacketResponder: BP-367880231-172.17.0.2-1733472062403:blk_1073741837_1013, type=LAST_IN_PIPELINE
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367)
	app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1779283c-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x66015df2-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x56b58936
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x03800144
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=509 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=546 (was 559), ProcessCount=11 (was 11), AvailableMemoryMB=4200 (was 4339)
2024-12-06T08:01:14,001 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=266, OpenFileDescriptor=509, MaxFileDescriptor=1048576, SystemLoadAverage=546, ProcessCount=11, AvailableMemoryMB=4199
2024-12-06T08:01:14,002 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25
2024-12-06T08:01:14,006 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:14,007 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6d4ed2ac to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2cf5982d
2024-12-06T08:01:14,012 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x6d4ed2ac {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6d4ed2ac to 127.0.0.1:64333
2024-12-06T08:01:14,012 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:14,026 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25
2024-12-06T08:01:14,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=26
2024-12-06T08:01:14,180 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-12-06T08:01:14,214 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:14,218 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26
2024-12-06T08:01:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=26
2024-12-06T08:01:14,222 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:14,226 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25
2024-12-06T08:01:14,226 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 196 msec
2024-12-06T08:01:14,226 INFO  [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-12-06T08:01:14,228 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 224 msec
2024-12-06T08:01:14,235 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:14,237 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x22d5fed8 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@30acc7b8
2024-12-06T08:01:14,240 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:14,240 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25
2024-12-06T08:01:14,326 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:14,327 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27
2024-12-06T08:01:14,331 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:14,333 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5815a909 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7eb25ccb
2024-12-06T08:01:14,336 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x5815a909 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5815a909 to 127.0.0.1:64333
2024-12-06T08:01:14,337 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:14,343 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27
2024-12-06T08:01:14,479 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-06T08:01:14,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=28
2024-12-06T08:01:14,497 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:14,499 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:14,499 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:14,499 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:14,499 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:14,502 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1de25d92 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d990f8b
2024-12-06T08:01:14,502 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:14,520 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:replication'
2024-12-06T08:01:14,522 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-06T08:01:14,526 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:14,526 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:14,526 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28
2024-12-06T08:01:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=28
2024-12-06T08:01:14,530 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:14,532 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27
2024-12-06T08:01:14,533 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 187 msec
2024-12-06T08:01:14,533 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:14,535 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 206 msec
2024-12-06T08:01:14,542 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:14,544 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1cc997fb to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@549e8cd4
2024-12-06T08:01:14,547 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:14,548 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27
2024-12-06T08:01:14,646 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:14,648 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-12-06T08:01:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29
2024-12-06T08:01:14,653 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:14,655 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2794b693 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d69b323
2024-12-06T08:01:14,658 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x2794b693 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2794b693 to 127.0.0.1:64333
2024-12-06T08:01:14,658 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:14,663 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29
2024-12-06T08:01:14,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=30
2024-12-06T08:01:14,817 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:14,818 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:14,818 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:14,818 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:14,818 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:14,820 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59a9fd5a to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@605511a3
2024-12-06T08:01:14,821 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:14,838 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:14,838 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:14,838 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30
2024-12-06T08:01:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=30
2024-12-06T08:01:14,842 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:14,845 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29
2024-12-06T08:01:14,845 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec
2024-12-06T08:01:14,845 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-12-06T08:01:14,847 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 197 msec
2024-12-06T08:01:14,856 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:14,859 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21092368 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@fec0893
2024-12-06T08:01:14,862 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:14,862 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29
2024-12-06T08:01:14,965 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:14,966 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false
2024-12-06T08:01:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31
2024-12-06T08:01:14,970 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:14,971 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b0f5bac to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@18679bef
2024-12-06T08:01:14,976 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x6b0f5bac {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6b0f5bac to 127.0.0.1:64333
2024-12-06T08:01:14,976 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:14,977 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:14,978 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=31, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:14,979 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=31, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=12 msec
2024-12-06T08:01:14,979 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31
2024-12-06T08:01:15,076 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
2024-12-06T08:01:15,077 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false
2024-12-06T08:01:15,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32
2024-12-06T08:01:15,080 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:15,081 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x60ed0116 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@52998015
2024-12-06T08:01:15,085 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x60ed0116 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x60ed0116 to 127.0.0.1:64333
2024-12-06T08:01:15,085 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:15,089 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32
2024-12-06T08:01:15,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=33
2024-12-06T08:01:15,243 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:15,244 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:15,244 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:15,244 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:15,245 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:15,247 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x290cb819 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c7fe829
2024-12-06T08:01:15,247 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:15,264 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:15,264 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:15,265 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33
2024-12-06T08:01:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=33
2024-12-06T08:01:15,268 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:15,270 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32
2024-12-06T08:01:15,270 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec
2024-12-06T08:01:15,270 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false
2024-12-06T08:01:15,272 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 194 msec
2024-12-06T08:01:15,282 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:15,283 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31ffd422 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@374ad61c
2024-12-06T08:01:15,286 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:15,286 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32
2024-12-06T08:01:15,396 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:15,397 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false
2024-12-06T08:01:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34
2024-12-06T08:01:15,400 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:15,401 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x15d037c5 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@74aa4bd1
2024-12-06T08:01:15,404 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x15d037c5 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x15d037c5 to 127.0.0.1:64333
2024-12-06T08:01:15,404 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:15,405 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:15,406 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=34, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:15,407 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=34, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=9 msec
2024-12-06T08:01:15,408 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34
2024-12-06T08:01:15,506 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
2024-12-06T08:01:15,507 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35
2024-12-06T08:01:15,511 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:15,512 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:15,512 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:15,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:15,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:15,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:15,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:15,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:15,517 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:15,575 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T08:01:15,576 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-06T08:01:15,577 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_replication
2024-12-06T08:01:15,577 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_replication Metrics about Tables on a single HBase RegionServer
2024-12-06T08:01:15,578 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T08:01:15,578 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-06T08:01:15,578 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-06T08:01:15,578 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-06T08:01:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35
2024-12-06T08:01:15,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=36
2024-12-06T08:01:15,670 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-12-06T08:01:15,670 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-12-06T08:01:15,670 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Replication stream was removed by a user
2024-12-06T08:01:15,670 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:15,671 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:15,672 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x62c6a58c to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@131f4b30
2024-12-06T08:01:15,672 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:15,673 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSourceManager(562): Done with the queue 1-aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:15,675 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=36
2024-12-06T08:01:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=36
2024-12-06T08:01:15,678 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:15,681 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35
2024-12-06T08:01:15,681 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 161 msec
2024-12-06T08:01:15,681 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-12-06T08:01:15,681 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-12-06T08:01:15,686 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-12-06T08:01:15,688 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 179 msec
2024-12-06T08:01:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35
2024-12-06T08:01:15,826 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:15,827 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37
2024-12-06T08:01:15,830 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:15,831 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=37, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:15,832 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=37, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-12-06T08:01:15,832 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37
2024-12-06T08:01:15,936 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-12-06T08:01:15,937 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38
2024-12-06T08:01:15,939 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:15,940 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=38, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:15,941 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=38, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-12-06T08:01:15,941 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38
2024-12-06T08:01:16,047 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:16,071 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=290 (was 266)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x21092368-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x21092368
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x62c6a58c-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1de25d92-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x59a9fd5a-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x62c6a58c-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x21092368-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x59a9fd5a-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1de25d92
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x290cb819-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x22d5fed8
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1cc997fb-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x31ffd422-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1cc997fb-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x290cb819-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x22d5fed8-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x59a9fd5a
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x22d5fed8-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x31ffd422-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1cc997fb
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1de25d92-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x31ffd422
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x62c6a58c
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x290cb819
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=540 (was 509) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=502 (was 546), ProcessCount=11 (was 11), AvailableMemoryMB=4177 (was 4199)
2024-12-06T08:01:16,081 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=290, OpenFileDescriptor=540, MaxFileDescriptor=1048576, SystemLoadAverage=502, ProcessCount=11, AvailableMemoryMB=4176
2024-12-06T08:01:16,083 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39
2024-12-06T08:01:16,088 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:16,090 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4fe45272 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@490ffa9c
2024-12-06T08:01:16,094 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x4fe45272 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4fe45272 to 127.0.0.1:64333
2024-12-06T08:01:16,094 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:16,106 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39
2024-12-06T08:01:16,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=40
2024-12-06T08:01:16,260 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-12-06T08:01:16,290 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:16,295 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40
2024-12-06T08:01:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=40
2024-12-06T08:01:16,298 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:16,301 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39
2024-12-06T08:01:16,301 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 192 msec
2024-12-06T08:01:16,301 INFO  [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-12-06T08:01:16,302 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 218 msec
2024-12-06T08:01:16,309 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:16,310 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7eb9a8d7 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@63ec4552
2024-12-06T08:01:16,313 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:16,313 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39
2024-12-06T08:01:16,406 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:16,406 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41
2024-12-06T08:01:16,410 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:16,411 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x76826811 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4485147f
2024-12-06T08:01:16,415 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x76826811 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x76826811 to 127.0.0.1:64333
2024-12-06T08:01:16,416 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:16,416 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:16,417 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=41, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:16,418 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=41, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=10 msec
2024-12-06T08:01:16,418 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41
2024-12-06T08:01:16,516 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists
2024-12-06T08:01:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-12-06T08:01:16,525 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42
2024-12-06T08:01:16,529 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:16,530 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=42, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:16,532 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=42, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec
2024-12-06T08:01:16,532 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42
2024-12-06T08:01:16,636 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-12-06T08:01:16,638 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=43, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43
2024-12-06T08:01:16,642 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test2'
2024-12-06T08:01:16,643 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x721c3d5c to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@70d2ead7
2024-12-06T08:01:16,646 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x721c3d5c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x721c3d5c to 127.0.0.1:64333
2024-12-06T08:01:16,646 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test2
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:16,661 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43
2024-12-06T08:01:16,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=44
2024-12-06T08:01:16,814 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=ADD_PEER
2024-12-06T08:01:16,844 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.ReplicationSource(231): queueId=2-aa9080fe83d2,46375,1733472065852, ReplicationSource: 2, currentBandwidth=0
2024-12-06T08:01:16,847 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=44
2024-12-06T08:01:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=44
2024-12-06T08:01:16,850 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for ADD on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:16,853 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43
2024-12-06T08:01:16,853 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec
2024-12-06T08:01:16,853 INFO  [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 2, config clusterKey=hbase+zk://127.0.0.1:64333/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-12-06T08:01:16,854 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 215 msec
2024-12-06T08:01:16,862 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,2-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test2'
2024-12-06T08:01:16,863 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,2-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x397280e5 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@78c94b8c
2024-12-06T08:01:16,866 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,2-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:16,866 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,2-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43
2024-12-06T08:01:16,956 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 2 completed
2024-12-06T08:01:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-12-06T08:01:16,957 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=45, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45
2024-12-06T08:01:16,961 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:16,962 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:16,962 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:16,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:16,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:16,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:16,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:16,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:16,967 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45
2024-12-06T08:01:17,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=46
2024-12-06T08:01:17,120 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-12-06T08:01:17,121 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-12-06T08:01:17,121 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Replication stream was removed by a user
2024-12-06T08:01:17,121 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:17,121 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:17,122 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ea69fe7 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3fe99002
2024-12-06T08:01:17,122 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:17,123 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSourceManager(562): Done with the queue 1-aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:17,126 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46
2024-12-06T08:01:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=46
2024-12-06T08:01:17,128 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:17,131 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=45
2024-12-06T08:01:17,131 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=45, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 161 msec
2024-12-06T08:01:17,131 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-12-06T08:01:17,131 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-12-06T08:01:17,137 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-12-06T08:01:17,139 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 180 msec
2024-12-06T08:01:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45
2024-12-06T08:01:17,276 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-12-06T08:01:17,278 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47
2024-12-06T08:01:17,283 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/sync-rep-state
2024-12-06T08:01:17,283 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state
2024-12-06T08:01:17,284 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/new-sync-rep-state
2024-12-06T08:01:17,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/new-sync-rep-state
2024-12-06T08:01:17,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2
2024-12-06T08:01:17,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state
2024-12-06T08:01:17,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/sync-rep-state
2024-12-06T08:01:17,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2
2024-12-06T08:01:17,290 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47
2024-12-06T08:01:17,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=48
2024-12-06T08:01:17,443 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=REMOVE_PEER
2024-12-06T08:01:17,443 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 2: 0
2024-12-06T08:01:17,443 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSource(687): peerId=2, Closing source 2-aa9080fe83d2,46375,1733472065852 because: Replication stream was removed by a user
2024-12-06T08:01:17,444 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,2-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=2, Interrupted while sleeping between retries
2024-12-06T08:01:17,444 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,2-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test2'
2024-12-06T08:01:17,445 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,2-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x790fd412 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@75fcdb9a
2024-12-06T08:01:17,445 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,2-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:17,446 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(562): Done with the queue 2-aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:17,449 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48
2024-12-06T08:01:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=48
2024-12-06T08:01:17,451 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for REMOVE on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:17,453 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47
2024-12-06T08:01:17,453 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 161 msec
2024-12-06T08:01:17,453 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 2, move on
2024-12-06T08:01:17,454 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 2, move on
2024-12-06T08:01:17,459 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 2
2024-12-06T08:01:17,461 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 181 msec
2024-12-06T08:01:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47
2024-12-06T08:01:17,596 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed
2024-12-06T08:01:17,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-12-06T08:01:17,597 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49
2024-12-06T08:01:17,600 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:17,601 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=49, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:17,602 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=49, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-12-06T08:01:17,602 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49
2024-12-06T08:01:17,706 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-12-06T08:01:17,706 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50
2024-12-06T08:01:17,709 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:17,710 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=50, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:17,711 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=50, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-12-06T08:01:17,711 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50
2024-12-06T08:01:17,816 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:17,833 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=302 (was 290)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x790fd412-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x7eb9a8d7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6ea69fe7-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x790fd412-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x397280e5-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x790fd412
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6ea69fe7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x7eb9a8d7-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x397280e5-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6ea69fe7-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x397280e5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x7eb9a8d7-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
 - Thread LEAK? -, OpenFileDescriptor=555 (was 540) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=502 (was 502), ProcessCount=11 (was 11), AvailableMemoryMB=4160 (was 4176)
2024-12-06T08:01:17,842 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=302, OpenFileDescriptor=555, MaxFileDescriptor=1048576, SystemLoadAverage=502, ProcessCount=11, AvailableMemoryMB=4160
2024-12-06T08:01:17,844 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=51, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51
2024-12-06T08:01:17,847 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: java.lang.ClassNotFoundException: whatever
	at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?]
	at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?]
	at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?]
	... 11 more
2024-12-06T08:01:17,848 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=51, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
Caused by: java.lang.ClassNotFoundException: whatever
	at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?]
	at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?]
	at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:17,849 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=51, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=5 msec
2024-12-06T08:01:17,849 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51
2024-12-06T08:01:17,956 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever
2024-12-06T08:01:17,961 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52
2024-12-06T08:01:17,963 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:17,964 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=52, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:17,965 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=52, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-12-06T08:01:17,965 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52
2024-12-06T08:01:18,066 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-12-06T08:01:18,066 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53
2024-12-06T08:01:18,069 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:18,070 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=53, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:18,071 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=53, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-12-06T08:01:18,071 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53
2024-12-06T08:01:18,176 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:18,193 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=302 (was 302), OpenFileDescriptor=555 (was 555), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=502 (was 502), ProcessCount=11 (was 11), AvailableMemoryMB=4156 (was 4160)
2024-12-06T08:01:18,202 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=302, OpenFileDescriptor=555, MaxFileDescriptor=1048576, SystemLoadAverage=502, ProcessCount=11, AvailableMemoryMB=4154
2024-12-06T08:01:18,203 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54
2024-12-06T08:01:18,207 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?]
	... 12 more
2024-12-06T08:01:18,208 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=54, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:18,209 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=54, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=6 msec
2024-12-06T08:01:18,209 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54
2024-12-06T08:01:18,316 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever
2024-12-06T08:01:18,317 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55
2024-12-06T08:01:18,319 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:18,320 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=55, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:18,321 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=55, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-12-06T08:01:18,321 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55
2024-12-06T08:01:18,426 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-12-06T08:01:18,427 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56
2024-12-06T08:01:18,429 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:18,430 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=56, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:18,431 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=56, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-12-06T08:01:18,431 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56
2024-12-06T08:01:18,536 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:18,552 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=302 (was 302), OpenFileDescriptor=555 (was 555), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=502 (was 502), ProcessCount=11 (was 11), AvailableMemoryMB=4153 (was 4154)
2024-12-06T08:01:18,561 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=302, OpenFileDescriptor=555, MaxFileDescriptor=1048576, SystemLoadAverage=502, ProcessCount=11, AvailableMemoryMB=4152
2024-12-06T08:01:18,562 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57
2024-12-06T08:01:18,565 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:18,566 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x55548b30 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@9a65a9f
2024-12-06T08:01:18,569 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x55548b30 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x55548b30 to 127.0.0.1:64333
2024-12-06T08:01:18,570 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:18,580 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57
2024-12-06T08:01:18,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=58
2024-12-06T08:01:18,733 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-12-06T08:01:18,763 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:18,765 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=58
2024-12-06T08:01:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=58
2024-12-06T08:01:18,768 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:18,770 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57
2024-12-06T08:01:18,770 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec
2024-12-06T08:01:18,770 INFO  [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-12-06T08:01:18,772 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 208 msec
2024-12-06T08:01:18,779 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:18,781 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3450c59c to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e7077de
2024-12-06T08:01:18,784 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:18,784 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57
2024-12-06T08:01:18,886 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:18,886 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59
2024-12-06T08:01:18,889 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:18,890 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1fdbb985 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@67c29107
2024-12-06T08:01:18,895 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x1fdbb985 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1fdbb985 to 127.0.0.1:64333
2024-12-06T08:01:18,895 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:18,899 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59
2024-12-06T08:01:19,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=60
2024-12-06T08:01:19,052 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:19,054 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:19,054 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:19,054 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:19,054 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:19,055 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6d9982c2 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6ef93927
2024-12-06T08:01:19,056 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:19,073 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:19,073 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:19,073 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60
2024-12-06T08:01:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=60
2024-12-06T08:01:19,076 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:19,078 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59
2024-12-06T08:01:19,078 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec
2024-12-06T08:01:19,079 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:19,080 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 193 msec
2024-12-06T08:01:19,088 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:19,090 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x739ac608 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d71c45c
2024-12-06T08:01:19,094 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:19,094 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59
2024-12-06T08:01:19,206 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:19,207 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false
2024-12-06T08:01:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61
2024-12-06T08:01:19,211 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:19,212 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69837cd7 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6110b15
2024-12-06T08:01:19,215 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x69837cd7 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69837cd7 to 127.0.0.1:64333
2024-12-06T08:01:19,215 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:19,219 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61
2024-12-06T08:01:19,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=62
2024-12-06T08:01:19,372 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:19,373 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:19,373 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:19,373 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:19,374 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:19,375 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d51575e to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@40049ba5
2024-12-06T08:01:19,375 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:19,394 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:19,394 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:19,394 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62
2024-12-06T08:01:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=62
2024-12-06T08:01:19,397 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:19,399 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61
2024-12-06T08:01:19,399 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 178 msec
2024-12-06T08:01:19,399 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false
2024-12-06T08:01:19,401 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 192 msec
2024-12-06T08:01:19,409 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:19,410 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x03362617 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8b946df
2024-12-06T08:01:19,413 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:19,413 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61
2024-12-06T08:01:19,526 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:19,526 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:19,527 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-12-06T08:01:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63
2024-12-06T08:01:19,530 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:19,531 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x473dd834 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61e941e7
2024-12-06T08:01:19,534 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x473dd834 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x473dd834 to 127.0.0.1:64333
2024-12-06T08:01:19,534 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:19,539 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63
2024-12-06T08:01:19,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=64
2024-12-06T08:01:19,692 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:19,693 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:19,693 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:19,693 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:19,693 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:19,695 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1f0ef537 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@57962588
2024-12-06T08:01:19,695 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:19,711 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:19,712 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:19,712 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=64
2024-12-06T08:01:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=64
2024-12-06T08:01:19,715 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:19,718 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63
2024-12-06T08:01:19,718 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 176 msec
2024-12-06T08:01:19,718 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-12-06T08:01:19,720 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 191 msec
2024-12-06T08:01:19,727 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:19,728 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b5480af to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8445921
2024-12-06T08:01:19,731 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:19,731 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:19,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63
2024-12-06T08:01:19,846 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:19,847 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:19,848 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65
2024-12-06T08:01:19,853 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:19,853 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:19,854 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:19,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:19,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:19,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:19,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:19,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:19,859 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65
2024-12-06T08:01:20,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=66
2024-12-06T08:01:20,012 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-12-06T08:01:20,013 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-12-06T08:01:20,013 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Replication stream was removed by a user
2024-12-06T08:01:20,013 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:20,013 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:20,014 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4251adba to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e390fd1
2024-12-06T08:01:20,014 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:20,015 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(562): Done with the queue 1-aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:20,018 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66
2024-12-06T08:01:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=66
2024-12-06T08:01:20,020 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:20,022 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65
2024-12-06T08:01:20,022 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 161 msec
2024-12-06T08:01:20,022 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-12-06T08:01:20,022 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-12-06T08:01:20,028 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-12-06T08:01:20,030 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 180 msec
2024-12-06T08:01:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65
2024-12-06T08:01:20,166 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:20,166 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67
2024-12-06T08:01:20,169 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:20,170 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=67, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:20,171 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=67, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-12-06T08:01:20,171 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67
2024-12-06T08:01:20,276 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-12-06T08:01:20,277 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68
2024-12-06T08:01:20,279 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:20,281 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=68, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:20,282 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=68, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-12-06T08:01:20,282 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68
2024-12-06T08:01:20,386 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:20,401 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=324 (was 302)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x3450c59c
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6d9982c2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x3450c59c-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x739ac608-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6d9982c2-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4251adba
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x03362617-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1f0ef537-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x03362617
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1f0ef537-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x739ac608
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x2b5480af
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x739ac608-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x0d51575e
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4251adba-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4251adba-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x03362617-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x0d51575e-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x0d51575e-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x1f0ef537
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x3450c59c-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x2b5480af-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x2b5480af-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6d9982c2-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
 - Thread LEAK? -, OpenFileDescriptor=585 (was 555) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=502 (was 502), ProcessCount=11 (was 11), AvailableMemoryMB=4137 (was 4152)
2024-12-06T08:01:20,411 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=324, OpenFileDescriptor=585, MaxFileDescriptor=1048576, SystemLoadAverage=502, ProcessCount=11, AvailableMemoryMB=4136
2024-12-06T08:01:20,412 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=69, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69
2024-12-06T08:01:20,415 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:20,417 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58d984e5 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@39844dff
2024-12-06T08:01:20,420 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x58d984e5 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x58d984e5 to 127.0.0.1:64333
2024-12-06T08:01:20,420 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:20,431 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69
2024-12-06T08:01:20,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=70
2024-12-06T08:01:20,584 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-12-06T08:01:20,615 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:20,618 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=70
2024-12-06T08:01:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=70
2024-12-06T08:01:20,620 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:20,622 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69
2024-12-06T08:01:20,622 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec
2024-12-06T08:01:20,622 INFO  [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-12-06T08:01:20,624 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 210 msec
2024-12-06T08:01:20,634 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:20,635 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5f18e5dc to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@375e552d
2024-12-06T08:01:20,638 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:20,638 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69
2024-12-06T08:01:20,736 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:20,737 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=71, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71
2024-12-06T08:01:20,740 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:20,741 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x780c1d94 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6b6e908
2024-12-06T08:01:20,747 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x780c1d94 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x780c1d94 to 127.0.0.1:64333
2024-12-06T08:01:20,747 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:20,751 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71
2024-12-06T08:01:20,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=72
2024-12-06T08:01:20,903 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:20,904 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:20,904 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:20,904 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:20,904 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:20,906 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4e89f239 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@96ec1ed
2024-12-06T08:01:20,906 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:20,924 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:20,924 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:20,924 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72
2024-12-06T08:01:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=72
2024-12-06T08:01:20,927 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:20,929 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=71
2024-12-06T08:01:20,929 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 176 msec
2024-12-06T08:01:20,929 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:20,931 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 193 msec
2024-12-06T08:01:20,942 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:20,944 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x56ff058f to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f9add30
2024-12-06T08:01:20,947 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:20,947 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71
2024-12-06T08:01:21,056 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:21,056 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:21,057 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false
2024-12-06T08:01:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=73, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73
2024-12-06T08:01:21,061 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:21,063 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7df9b66b to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5788852a
2024-12-06T08:01:21,066 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x7df9b66b {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7df9b66b to 127.0.0.1:64333
2024-12-06T08:01:21,066 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:21,072 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:21,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73
2024-12-06T08:01:21,174 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-06T08:01:21,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=74
2024-12-06T08:01:21,224 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:21,226 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:21,226 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:21,226 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:21,226 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:21,228 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x572f5e25 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6005dced
2024-12-06T08:01:21,228 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:21,246 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:21,246 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:21,246 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74
2024-12-06T08:01:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=74
2024-12-06T08:01:21,249 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:21,251 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73
2024-12-06T08:01:21,252 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec
2024-12-06T08:01:21,252 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false
2024-12-06T08:01:21,253 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 195 msec
2024-12-06T08:01:21,262 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:21,263 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f80f580 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@56da4a58
2024-12-06T08:01:21,267 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:21,267 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73
2024-12-06T08:01:21,376 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:21,377 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:21,378 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:21,379 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-12-06T08:01:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=75, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75
2024-12-06T08:01:21,382 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:21,383 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4ac899f6 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1cf89a92
2024-12-06T08:01:21,395 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x4ac899f6 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4ac899f6 to 127.0.0.1:64333
2024-12-06T08:01:21,395 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:21,412 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75
2024-12-06T08:01:21,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=76
2024-12-06T08:01:21,565 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:21,567 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:21,567 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:21,567 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:21,567 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:21,569 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x399e41b7 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@592d5997
2024-12-06T08:01:21,569 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:21,587 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:21,587 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:21,587 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76
2024-12-06T08:01:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=76
2024-12-06T08:01:21,590 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:21,592 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75
2024-12-06T08:01:21,592 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 178 msec
2024-12-06T08:01:21,592 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-12-06T08:01:21,594 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 213 msec
2024-12-06T08:01:21,602 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:21,604 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6e9cf834 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@57f88aa8
2024-12-06T08:01:21,607 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:21,607 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75
2024-12-06T08:01:21,696 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:21,696 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:21,698 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:21,699 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-12-06T08:01:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=77, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77
2024-12-06T08:01:21,703 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:21,704 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x46408c35 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@68b08c55
2024-12-06T08:01:21,712 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x46408c35 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x46408c35 to 127.0.0.1:64333
2024-12-06T08:01:21,712 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:21,718 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77
2024-12-06T08:01:21,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=78
2024-12-06T08:01:21,871 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:21,872 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:21,872 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:21,872 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:21,872 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:21,874 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6082f3ae to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@214e8427
2024-12-06T08:01:21,874 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:21,892 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:21,892 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:21,892 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78
2024-12-06T08:01:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=78
2024-12-06T08:01:21,895 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:21,898 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77
2024-12-06T08:01:21,898 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 178 msec
2024-12-06T08:01:21,898 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-12-06T08:01:21,900 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 199 msec
2024-12-06T08:01:21,908 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:21,910 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6401d8d4 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@23824a1f
2024-12-06T08:01:21,914 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:21,914 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77
2024-12-06T08:01:22,016 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:22,017 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:22,018 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:22,019 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-12-06T08:01:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79
2024-12-06T08:01:22,022 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:22,023 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ea11f9f to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3441364c
2024-12-06T08:01:22,027 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x6ea11f9f {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6ea11f9f to 127.0.0.1:64333
2024-12-06T08:01:22,027 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:22,030 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79
2024-12-06T08:01:22,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=80
2024-12-06T08:01:22,183 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:22,185 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:22,185 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:22,185 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:22,185 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:22,186 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6cb68f69 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@ddfa42d
2024-12-06T08:01:22,187 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:22,204 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:22,204 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:22,204 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80
2024-12-06T08:01:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=80
2024-12-06T08:01:22,207 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:22,210 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79
2024-12-06T08:01:22,210 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec
2024-12-06T08:01:22,210 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-12-06T08:01:22,211 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 191 msec
2024-12-06T08:01:22,220 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:22,221 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x479b6aff to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@199e6afc
2024-12-06T08:01:22,224 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:22,224 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79
2024-12-06T08:01:22,336 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:22,336 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:22,337 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:22,338 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-12-06T08:01:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81
2024-12-06T08:01:22,342 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:22,343 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x30f5ceb1 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4c2f7da6
2024-12-06T08:01:22,346 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x30f5ceb1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x30f5ceb1 to 127.0.0.1:64333
2024-12-06T08:01:22,346 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:22,350 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81
2024-12-06T08:01:22,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=82
2024-12-06T08:01:22,503 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:22,504 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:22,505 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:22,505 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:22,505 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:22,506 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4f302e4f to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7dd936a0
2024-12-06T08:01:22,507 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:22,525 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:22,525 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:22,525 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82
2024-12-06T08:01:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=82
2024-12-06T08:01:22,529 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:22,531 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81
2024-12-06T08:01:22,531 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec
2024-12-06T08:01:22,531 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-12-06T08:01:22,533 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 193 msec
2024-12-06T08:01:22,540 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:22,542 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x24902694 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@277397e2
2024-12-06T08:01:22,545 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:22,545 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81
2024-12-06T08:01:22,656 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:22,657 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:22,658 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-12-06T08:01:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83
2024-12-06T08:01:22,661 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:22,662 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x47e89967 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d9375ec
2024-12-06T08:01:22,665 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x47e89967 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x47e89967 to 127.0.0.1:64333
2024-12-06T08:01:22,665 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:22,670 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83
2024-12-06T08:01:22,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=84
2024-12-06T08:01:22,823 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:22,825 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84
2024-12-06T08:01:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=84
2024-12-06T08:01:22,827 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:22,829 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83
2024-12-06T08:01:22,829 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 157 msec
2024-12-06T08:01:22,829 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-12-06T08:01:22,831 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 171 msec
2024-12-06T08:01:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83
2024-12-06T08:01:22,976 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:22,976 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:22,977 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:22,978 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false
2024-12-06T08:01:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85
2024-12-06T08:01:22,981 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:22,982 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x09f4c6ae to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7c6f4ea2
2024-12-06T08:01:22,986 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x09f4c6ae {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x09f4c6ae to 127.0.0.1:64333
2024-12-06T08:01:22,986 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:22,991 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85
2024-12-06T08:01:23,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=86
2024-12-06T08:01:23,144 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:23,145 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:23,145 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:23,145 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:23,145 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:23,147 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4a62a85e to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5b31397d
2024-12-06T08:01:23,147 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:23,165 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:23,165 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:23,165 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86
2024-12-06T08:01:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=86
2024-12-06T08:01:23,168 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:23,170 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85
2024-12-06T08:01:23,170 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec
2024-12-06T08:01:23,170 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false
2024-12-06T08:01:23,172 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 192 msec
2024-12-06T08:01:23,187 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:23,189 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x079650c1 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5b383f04
2024-12-06T08:01:23,193 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:23,194 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85
2024-12-06T08:01:23,296 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:23,297 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:23,298 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false
2024-12-06T08:01:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87
2024-12-06T08:01:23,302 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:23,303 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2ef22619 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@31006c20
2024-12-06T08:01:23,306 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x2ef22619 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2ef22619 to 127.0.0.1:64333
2024-12-06T08:01:23,306 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:23,311 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87
2024-12-06T08:01:23,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=88
2024-12-06T08:01:23,464 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:23,465 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:23,465 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:23,465 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:23,465 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:23,467 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69df80da to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@339504b7
2024-12-06T08:01:23,467 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:23,492 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:23,493 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:23,493 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88
2024-12-06T08:01:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=88
2024-12-06T08:01:23,496 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:23,498 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87
2024-12-06T08:01:23,498 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 185 msec
2024-12-06T08:01:23,498 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false
2024-12-06T08:01:23,500 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 200 msec
2024-12-06T08:01:23,508 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:23,510 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x564f483c to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e318599
2024-12-06T08:01:23,513 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:23,513 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87
2024-12-06T08:01:23,616 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:23,616 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:23,617 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89
2024-12-06T08:01:23,622 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:23,622 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:23,623 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:23,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:23,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:23,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:23,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:23,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:23,629 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89
2024-12-06T08:01:23,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=90
2024-12-06T08:01:23,783 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-12-06T08:01:23,783 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-12-06T08:01:23,783 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Replication stream was removed by a user
2024-12-06T08:01:23,783 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:23,783 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:23,785 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5a51960b to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@60467e95
2024-12-06T08:01:23,785 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:23,785 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(562): Done with the queue 1-aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:23,788 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90
2024-12-06T08:01:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=90
2024-12-06T08:01:23,791 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:23,792 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89
2024-12-06T08:01:23,792 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 162 msec
2024-12-06T08:01:23,793 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-12-06T08:01:23,793 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-12-06T08:01:23,798 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-12-06T08:01:23,799 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 181 msec
2024-12-06T08:01:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89
2024-12-06T08:01:23,936 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:23,936 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91
2024-12-06T08:01:23,938 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:23,939 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=91, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:23,940 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=91, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-12-06T08:01:23,940 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91
2024-12-06T08:01:24,046 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-12-06T08:01:24,046 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92
2024-12-06T08:01:24,049 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:24,050 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=92, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:24,051 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=92, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-12-06T08:01:24,051 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92
2024-12-06T08:01:24,156 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:24,173 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=377 (was 324)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6e9cf834
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6082f3ae-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4e89f239-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x2f80f580-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x56ff058f-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x399e41b7-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4f302e4f-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x564f483c-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x5f18e5dc-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x479b6aff
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x079650c1-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x479b6aff-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4a62a85e
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4f302e4f-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6401d8d4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x56ff058f
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x399e41b7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x5f18e5dc-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x69df80da
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x572f5e25-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x24902694-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6401d8d4-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x2f80f580-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6082f3ae-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6e9cf834-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6cb68f69-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6cb68f69-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x69df80da-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x399e41b7-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6401d8d4-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x079650c1-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x56ff058f-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x5f18e5dc
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x572f5e25-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x5a51960b-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x69df80da-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x5a51960b-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x564f483c
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4a62a85e-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6e9cf834-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6082f3ae
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4e89f239
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x079650c1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x2f80f580
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x572f5e25
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x5a51960b
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x24902694
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4a62a85e-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x479b6aff-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4f302e4f
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x6cb68f69
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x564f483c-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x24902694-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4e89f239-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
 - Thread LEAK? -, OpenFileDescriptor=653 (was 585) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=486 (was 502), ProcessCount=11 (was 11), AvailableMemoryMB=4021 (was 4136)
2024-12-06T08:01:24,184 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=377, OpenFileDescriptor=653, MaxFileDescriptor=1048576, SystemLoadAverage=486, ProcessCount=11, AvailableMemoryMB=4020
2024-12-06T08:01:24,185 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93
2024-12-06T08:01:24,188 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:24,189 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x162c719e to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@321aad60
2024-12-06T08:01:24,191 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x162c719e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x162c719e to 127.0.0.1:64333
2024-12-06T08:01:24,192 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:24,202 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93
2024-12-06T08:01:24,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=94
2024-12-06T08:01:24,354 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-12-06T08:01:24,390 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:24,394 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94
2024-12-06T08:01:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=94
2024-12-06T08:01:24,397 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:24,399 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93
2024-12-06T08:01:24,399 INFO  [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-12-06T08:01:24,399 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 195 msec
2024-12-06T08:01:24,401 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 214 msec
2024-12-06T08:01:24,414 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:24,415 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x65277045 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55a188ea
2024-12-06T08:01:24,419 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:24,419 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93
2024-12-06T08:01:24,506 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-12-06T08:01:24,511 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3992): Client=jenkins//172.17.0.2 disable replication peer, id=1
2024-12-06T08:01:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure
2024-12-06T08:01:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95
2024-12-06T08:01:24,517 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95
2024-12-06T08:01:24,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=96
2024-12-06T08:01:24,670 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=DISABLE_PEER
2024-12-06T08:01:24,671 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:24,671 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:24,671 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:24,671 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:24,673 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0df8c4c3 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@334b43ce
2024-12-06T08:01:24,673 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:24,691 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:24,691 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:24,691 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96
2024-12-06T08:01:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=96
2024-12-06T08:01:24,694 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for DISABLE on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:24,695 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95
2024-12-06T08:01:24,695 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec
2024-12-06T08:01:24,695 INFO  [PEWorker-3 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1
2024-12-06T08:01:24,697 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 185 msec
2024-12-06T08:01:24,708 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:24,710 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2d90ec8a to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1caf1316
2024-12-06T08:01:24,713 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:24,713 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95
2024-12-06T08:01:24,836 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-12-06T08:01:24,837 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97
2024-12-06T08:01:24,841 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:24,841 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:24,842 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:24,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:24,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:24,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:24,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:24,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:24,845 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97
2024-12-06T08:01:24,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=98
2024-12-06T08:01:24,998 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-12-06T08:01:24,998 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-12-06T08:01:24,998 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Replication stream was removed by a user
2024-12-06T08:01:24,998 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:24,998 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:25,001 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4b5dee4f to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@29f92d09
2024-12-06T08:01:25,001 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:25,001 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(562): Done with the queue 1-aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:25,004 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98
2024-12-06T08:01:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=98
2024-12-06T08:01:25,006 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:25,009 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97
2024-12-06T08:01:25,009 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 161 msec
2024-12-06T08:01:25,009 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-12-06T08:01:25,009 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-12-06T08:01:25,015 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-12-06T08:01:25,017 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 178 msec
2024-12-06T08:01:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97
2024-12-06T08:01:25,156 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:25,157 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:25,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99
2024-12-06T08:01:25,159 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:25,160 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=99, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:25,161 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=99, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-12-06T08:01:25,161 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99
2024-12-06T08:01:25,267 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-12-06T08:01:25,267 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100
2024-12-06T08:01:25,269 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:25,270 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=100, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:25,271 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=100, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-12-06T08:01:25,271 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100
2024-12-06T08:01:25,376 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:25,394 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=389 (was 377)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x2d90ec8a-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x65277045-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4b5dee4f
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x2d90ec8a-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x65277045
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x0df8c4c3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x0df8c4c3-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4b5dee4f-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x4b5dee4f-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x0df8c4c3-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x65277045-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x2d90ec8a
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=669 (was 653) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=486 (was 486), ProcessCount=11 (was 11), AvailableMemoryMB=3992 (was 4020)
2024-12-06T08:01:25,405 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=389, OpenFileDescriptor=669, MaxFileDescriptor=1048576, SystemLoadAverage=486, ProcessCount=11, AvailableMemoryMB=3990
2024-12-06T08:01:25,406 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=dummy_peer
2024-12-06T08:01:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101
2024-12-06T08:01:25,408 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:25,409 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=101, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:25,410 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=101, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-12-06T08:01:25,410 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101
2024-12-06T08:01:25,516 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer
2024-12-06T08:01:25,517 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=102, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102
2024-12-06T08:01:25,519 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:25,520 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=102, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:25,521 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=102, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-12-06T08:01:25,521 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102
2024-12-06T08:01:25,626 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-12-06T08:01:25,626 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103
2024-12-06T08:01:25,629 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:25,630 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=103, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:25,631 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=103, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-12-06T08:01:25,631 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103
2024-12-06T08:01:25,736 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:25,753 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=389 (was 389), OpenFileDescriptor=669 (was 669), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=486 (was 486), ProcessCount=11 (was 11), AvailableMemoryMB=3993 (was 3990) - AvailableMemoryMB LEAK? -
2024-12-06T08:01:25,763 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=389, OpenFileDescriptor=669, MaxFileDescriptor=1048576, SystemLoadAverage=486, ProcessCount=11, AvailableMemoryMB=3993
2024-12-06T08:01:25,765 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104
2024-12-06T08:01:25,768 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:25,770 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d900296 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6bab52b1
2024-12-06T08:01:25,774 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x0d900296 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0d900296 to 127.0.0.1:64333
2024-12-06T08:01:25,774 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:25,784 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104
2024-12-06T08:01:25,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=105
2024-12-06T08:01:25,938 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-12-06T08:01:25,969 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:25,972 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=105
2024-12-06T08:01:25,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=105
2024-12-06T08:01:25,975 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:25,976 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104
2024-12-06T08:01:25,976 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec
2024-12-06T08:01:25,977 INFO  [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-12-06T08:01:25,978 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 212 msec
2024-12-06T08:01:25,989 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:25,990 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x73c9735c to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@625b6e25
2024-12-06T08:01:25,996 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:25,996 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104
2024-12-06T08:01:26,086 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null
2024-12-06T08:01:26,087 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106
2024-12-06T08:01:26,091 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:26,091 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:26,092 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:26,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:26,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:26,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:26,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:26,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:26,096 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106
2024-12-06T08:01:26,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=107
2024-12-06T08:01:26,249 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-12-06T08:01:26,249 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-12-06T08:01:26,249 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Replication stream was removed by a user
2024-12-06T08:01:26,249 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:26,250 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:26,251 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x62df8b56 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@49761d3e
2024-12-06T08:01:26,251 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:26,252 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSourceManager(562): Done with the queue 1-aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:26,254 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107
2024-12-06T08:01:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=107
2024-12-06T08:01:26,256 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:26,258 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106
2024-12-06T08:01:26,258 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 160 msec
2024-12-06T08:01:26,258 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-12-06T08:01:26,258 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-12-06T08:01:26,264 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-12-06T08:01:26,266 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 178 msec
2024-12-06T08:01:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106
2024-12-06T08:01:26,406 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:26,406 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108
2024-12-06T08:01:26,408 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:26,409 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=108, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:26,410 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=108, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-12-06T08:01:26,410 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108
2024-12-06T08:01:26,516 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-12-06T08:01:26,517 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109
2024-12-06T08:01:26,519 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:26,520 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=109, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:26,521 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=109, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-12-06T08:01:26,521 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109
2024-12-06T08:01:26,626 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:26,642 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=395 (was 389)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x73c9735c
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x62df8b56-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x62df8b56-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x62df8b56
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x73c9735c-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x73c9735c-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
 - Thread LEAK? -, OpenFileDescriptor=677 (was 669) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=463 (was 486), ProcessCount=11 (was 11), AvailableMemoryMB=3974 (was 3993)
2024-12-06T08:01:26,653 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=395, OpenFileDescriptor=677, MaxFileDescriptor=1048576, SystemLoadAverage=463, ProcessCount=11, AvailableMemoryMB=3973
2024-12-06T08:01:26,659 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] ipc.MetricsHBaseServer(152): Unknown exception type
java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] ipc.CallRunner(138): callId: 307 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.2:33194 deadline: 1733472146659, exception=java.io.IOException: Replication peer modification disabled
2024-12-06T08:01:26,666 WARN  [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 4 ms
java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?]
	at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 32 more
2024-12-06T08:01:26,775 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] ipc.MetricsHBaseServer(152): Unknown exception type
java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] ipc.CallRunner(138): callId: 308 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.2:33194 deadline: 1733472146775, exception=java.io.IOException: Replication peer modification disabled
2024-12-06T08:01:26,776 WARN  [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 117 ms
java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?]
	at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 32 more
2024-12-06T08:01:26,777 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions:
2024-12-06T08:01:26.667Z, java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T08:01:26.776Z, java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)


2024-12-06T08:01:26,779 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110
2024-12-06T08:01:26,781 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:26,782 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=110, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:26,783 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=110, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-12-06T08:01:26,783 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110
2024-12-06T08:01:26,886 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-12-06T08:01:26,886 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111
2024-12-06T08:01:26,888 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:26,889 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=111, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:26,890 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=111, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-12-06T08:01:26,890 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111
2024-12-06T08:01:26,996 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:27,013 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=395 (was 395), OpenFileDescriptor=677 (was 677), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=463 (was 463), ProcessCount=11 (was 11), AvailableMemoryMB=3969 (was 3973)
2024-12-06T08:01:27,023 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=395, OpenFileDescriptor=677, MaxFileDescriptor=1048576, SystemLoadAverage=463, ProcessCount=11, AvailableMemoryMB=3969
2024-12-06T08:01:27,024 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112
2024-12-06T08:01:27,027 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:27,028 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x396b5d3c to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4e38f15f
2024-12-06T08:01:27,031 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x396b5d3c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x396b5d3c to 127.0.0.1:64333
2024-12-06T08:01:27,031 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:27,046 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112
2024-12-06T08:01:27,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=113
2024-12-06T08:01:27,198 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-12-06T08:01:27,229 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:27,232 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=113
2024-12-06T08:01:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=113
2024-12-06T08:01:27,234 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:27,236 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112
2024-12-06T08:01:27,236 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec
2024-12-06T08:01:27,236 INFO  [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-12-06T08:01:27,237 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 212 msec
2024-12-06T08:01:27,246 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:27,248 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5408e30e to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f78e444
2024-12-06T08:01:27,251 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:27,251 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112
2024-12-06T08:01:27,345 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:27,346 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:27,347 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114
2024-12-06T08:01:27,350 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:27,351 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:27,351 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:27,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:27,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:27,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:27,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:27,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:27,355 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114
2024-12-06T08:01:27,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=115
2024-12-06T08:01:27,507 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-12-06T08:01:27,507 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-12-06T08:01:27,507 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Replication stream was removed by a user
2024-12-06T08:01:27,508 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:27,508 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:27,509 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x607c121a to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@58ac5afa
2024-12-06T08:01:27,510 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:27,510 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSourceManager(562): Done with the queue 1-aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:27,512 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=115
2024-12-06T08:01:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=115
2024-12-06T08:01:27,514 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:27,516 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114
2024-12-06T08:01:27,516 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-12-06T08:01:27,516 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec
2024-12-06T08:01:27,516 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-12-06T08:01:27,521 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-12-06T08:01:27,522 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 174 msec
2024-12-06T08:01:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114
2024-12-06T08:01:27,666 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:27,666 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116
2024-12-06T08:01:27,668 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:27,669 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=116, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:27,670 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=116, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-12-06T08:01:27,670 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116
2024-12-06T08:01:27,776 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:27,793 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=401 (was 395)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x607c121a-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x5408e30e-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x607c121a
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x5408e30e-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x607c121a-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x5408e30e
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=685 (was 677) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=463 (was 463), ProcessCount=11 (was 11), AvailableMemoryMB=3953 (was 3969)
2024-12-06T08:01:27,804 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=401, OpenFileDescriptor=685, MaxFileDescriptor=1048576, SystemLoadAverage=463, ProcessCount=11, AvailableMemoryMB=3952
2024-12-06T08:01:27,805 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117
2024-12-06T08:01:27,807 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:27,808 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4b801fe0 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@60bc2267
2024-12-06T08:01:27,811 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x4b801fe0 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4b801fe0 to 127.0.0.1:64333
2024-12-06T08:01:27,811 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:27,821 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117
2024-12-06T08:01:27,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=118
2024-12-06T08:01:27,974 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-12-06T08:01:28,005 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:28,007 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118
2024-12-06T08:01:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=118
2024-12-06T08:01:28,009 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:28,011 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117
2024-12-06T08:01:28,011 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec
2024-12-06T08:01:28,011 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-12-06T08:01:28,012 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 206 msec
2024-12-06T08:01:28,022 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:28,023 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x07ac0126 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@149f0fec
2024-12-06T08:01:28,026 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:28,026 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117
2024-12-06T08:01:28,126 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:28,127 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:28,127 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false
2024-12-06T08:01:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119
2024-12-06T08:01:28,130 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:28,131 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3145aefe to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@ad9c510
2024-12-06T08:01:28,134 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x3145aefe {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3145aefe to 127.0.0.1:64333
2024-12-06T08:01:28,135 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:28,139 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119
2024-12-06T08:01:28,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=120
2024-12-06T08:01:28,291 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:28,292 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120
2024-12-06T08:01:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=120
2024-12-06T08:01:28,294 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:28,296 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119
2024-12-06T08:01:28,296 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 155 msec
2024-12-06T08:01:28,296 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false
2024-12-06T08:01:28,297 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 169 msec
2024-12-06T08:01:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119
2024-12-06T08:01:28,445 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:28,446 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:28,447 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121
2024-12-06T08:01:28,450 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:28,450 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:28,451 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:28,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:28,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:28,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:28,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:28,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:28,454 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121
2024-12-06T08:01:28,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=122
2024-12-06T08:01:28,607 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-12-06T08:01:28,607 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-12-06T08:01:28,607 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Replication stream was removed by a user
2024-12-06T08:01:28,607 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:28,607 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:28,609 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59a08698 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ca60513
2024-12-06T08:01:28,609 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:28,609 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSourceManager(562): Done with the queue 1-aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:28,612 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122
2024-12-06T08:01:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=122
2024-12-06T08:01:28,614 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:28,615 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121
2024-12-06T08:01:28,615 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 160 msec
2024-12-06T08:01:28,615 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-12-06T08:01:28,615 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-12-06T08:01:28,620 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-12-06T08:01:28,621 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 173 msec
2024-12-06T08:01:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121
2024-12-06T08:01:28,766 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:28,766 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123
2024-12-06T08:01:28,768 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:28,769 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=123, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:28,770 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=123, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-12-06T08:01:28,770 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123
2024-12-06T08:01:28,876 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-12-06T08:01:28,876 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124
2024-12-06T08:01:28,878 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:28,879 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=124, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:28,880 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=124, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-12-06T08:01:28,880 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124
2024-12-06T08:01:28,986 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:29,008 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=406 (was 401)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x07ac0126-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x59a08698-SendThread(127.0.0.1:64333)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x07ac0126-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x07ac0126
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x59a08698
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64333@0x59a08698-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
 - Thread LEAK? -, OpenFileDescriptor=693 (was 685) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=463 (was 463), ProcessCount=11 (was 11), AvailableMemoryMB=3924 (was 3952)
2024-12-06T08:01:29,022 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=406, OpenFileDescriptor=693, MaxFileDescriptor=1048576, SystemLoadAverage=463, ProcessCount=11, AvailableMemoryMB=3922
2024-12-06T08:01:29,024 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125
2024-12-06T08:01:29,037 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125
2024-12-06T08:01:29,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=126
2024-12-06T08:01:29,189 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-12-06T08:01:29,220 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:29,223 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126
2024-12-06T08:01:29,223 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(613): peerId=1, queueId=1-aa9080fe83d2,46375,1733472065852 (queues=1) is replicating from cluster=c9b453b4-ba9a-4f5f-80a1-c9cf6cbc0ebc to cluster=c9b453b4-ba9a-4f5f-80a1-c9cf6cbc0ebc
2024-12-06T08:01:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=126
2024-12-06T08:01:29,224 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(382): peerId=1, starting shipping worker for walGroupId=aa9080fe83d2%2C46375%2C1733472065852
2024-12-06T08:01:29,225 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:29,227 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125
2024-12-06T08:01:29,227 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec
2024-12-06T08:01:29,227 INFO  [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false
2024-12-06T08:01:29,228 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 203 msec
2024-12-06T08:01:29,229 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=1-aa9080fe83d2,46375,1733472065852, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1
2024-12-06T08:01:29,231 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852.replicationSource.shipperaa9080fe83d2%2C46375%2C1733472065852,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: aa9080fe83d2%2C46375%2C1733472065852
2024-12-06T08:01:29,235 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852.replicationSource.wal-reader.aa9080fe83d2%2C46375%2C1733472065852,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/WALs/aa9080fe83d2,46375,1733472065852/aa9080fe83d2%2C46375%2C1733472065852.1733472068425, startPosition=0, beingWritten=true
2024-12-06T08:01:29,287 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-06T08:01:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125
2024-12-06T08:01:29,346 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:29,347 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127
2024-12-06T08:01:29,349 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: 
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: java.io.IOException: Cluster key passed  is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?]
	... 12 more
2024-12-06T08:01:29,350 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=127, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: 
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: 
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
Caused by: java.io.IOException: Cluster key passed  is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:29,351 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=127, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec
2024-12-06T08:01:29,351 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127
2024-12-06T08:01:29,456 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 
2024-12-06T08:01:29,457 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-12-06T08:01:29,460 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:29,461 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:29,461 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:29,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:29,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:29,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:29,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:29,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:29,465 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-12-06T08:01:29,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=129
2024-12-06T08:01:29,617 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-12-06T08:01:29,617 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-12-06T08:01:29,617 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Replication stream was removed by a user
2024-12-06T08:01:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-12-06T08:01:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-12-06T08:01:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-12-06T08:01:30,618 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852.replicationSource.wal-reader.aa9080fe83d2%2C46375%2C1733472065852,1-aa9080fe83d2,46375,1733472065852 {}] util.Threads(127): sleep interrupted
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) ~[classes/:?]
2024-12-06T08:01:30,618 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852.replicationSource.shipperaa9080fe83d2%2C46375%2C1733472065852,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch
java.lang.InterruptedException: null
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?]
	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:309) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?]
2024-12-06T08:01:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-12-06T08:01:31,618 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSource(739): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852.replicationSource.shipperaa9080fe83d2%2C46375%2C1733472065852,1-aa9080fe83d2,46375,1733472065852 terminated
2024-12-06T08:01:31,618 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSourceManager(562): Done with the queue 1-aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:31,620 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=129
2024-12-06T08:01:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=129
2024-12-06T08:01:31,622 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:31,624 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128
2024-12-06T08:01:31,624 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-12-06T08:01:31,624 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-12-06T08:01:31,624 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1570 sec
2024-12-06T08:01:31,629 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-12-06T08:01:31,631 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.1720 sec
2024-12-06T08:01:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-12-06T08:01:33,616 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:33,616 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2
2024-12-06T08:01:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130
2024-12-06T08:01:33,618 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T08:01:33,619 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=130, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-12-06T08:01:33,620 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=130, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-12-06T08:01:33,620 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-12-06T08:01:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130
2024-12-06T08:01:33,726 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-12-06T08:01:33,742 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=408 (was 406) - Thread LEAK? -, OpenFileDescriptor=698 (was 693) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 463), ProcessCount=11 (was 11), AvailableMemoryMB=3883 (was 3922)
2024-12-06T08:01:33,753 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=408, OpenFileDescriptor=698, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=3882
2024-12-06T08:01:33,755 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-12-06T08:01:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-12-06T08:01:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131
2024-12-06T08:01:33,758 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:33,759 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59cb578a to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@129288bb
2024-12-06T08:01:33,762 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x59cb578a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x59cb578a to 127.0.0.1:64333
2024-12-06T08:01:33,762 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:33,776 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131
2024-12-06T08:01:33,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=132
2024-12-06T08:01:33,929 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-12-06T08:01:33,960 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:33,962 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132
2024-12-06T08:01:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=132
2024-12-06T08:01:33,965 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:33,966 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131
2024-12-06T08:01:33,967 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec
2024-12-06T08:01:33,967 INFO  [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-12-06T08:01:33,968 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 212 msec
2024-12-06T08:01:33,978 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:33,979 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5fc5b243 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4fe91d3
2024-12-06T08:01:33,982 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:33,982 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131
2024-12-06T08:01:34,075 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:34,076 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133
2024-12-06T08:01:34,079 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:34,080 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x669304be to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ab15e2b
2024-12-06T08:01:34,083 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x669304be {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x669304be to 127.0.0.1:64333
2024-12-06T08:01:34,083 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:34,087 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133
2024-12-06T08:01:34,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=134
2024-12-06T08:01:34,240 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:34,241 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:34,241 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:34,241 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:34,241 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:34,243 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x19255325 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@a14278
2024-12-06T08:01:34,243 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:34,261 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:34,261 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:34,261 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134
2024-12-06T08:01:34,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=134
2024-12-06T08:01:34,263 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:34,264 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:01:34,265 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133
2024-12-06T08:01:34,265 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 176 msec
2024-12-06T08:01:34,265 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:34,266 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 189 msec
2024-12-06T08:01:34,277 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:34,278 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x67e378ba to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2a655101
2024-12-06T08:01:34,281 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:34,281 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:34,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133
2024-12-06T08:01:34,396 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:34,396 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:34,397 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:34,398 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:34,399 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false
2024-12-06T08:01:34,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:34,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135
2024-12-06T08:01:34,401 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:34,402 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6bf1c69b to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a090206
2024-12-06T08:01:34,405 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x6bf1c69b {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6bf1c69b to 127.0.0.1:64333
2024-12-06T08:01:34,406 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:34,410 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135
2024-12-06T08:01:34,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=136
2024-12-06T08:01:34,563 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:34,564 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:34,564 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:34,564 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:34,564 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:34,566 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a17d2ee to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1bde2469
2024-12-06T08:01:34,566 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:34,583 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:34,583 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:34,583 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136
2024-12-06T08:01:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=136
2024-12-06T08:01:34,585 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:34,587 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135
2024-12-06T08:01:34,587 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 175 msec
2024-12-06T08:01:34,587 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false
2024-12-06T08:01:34,588 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 188 msec
2024-12-06T08:01:34,598 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:34,599 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4b49370a to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2a96e66a
2024-12-06T08:01:34,602 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:34,602 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135
2024-12-06T08:01:34,716 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:34,716 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:34,717 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:34,718 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:34,719 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:34,719 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false
2024-12-06T08:01:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137
2024-12-06T08:01:34,722 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:34,723 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x23da79d8 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@39f7ced0
2024-12-06T08:01:34,728 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x23da79d8 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x23da79d8 to 127.0.0.1:64333
2024-12-06T08:01:34,728 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:34,731 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137
2024-12-06T08:01:34,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=138
2024-12-06T08:01:34,884 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:34,885 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:34,885 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:34,885 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:34,886 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:34,887 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x15b31be5 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@18be487b
2024-12-06T08:01:34,887 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:34,905 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:34,905 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:34,905 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138
2024-12-06T08:01:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=138
2024-12-06T08:01:34,907 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:34,909 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137
2024-12-06T08:01:34,909 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 176 msec
2024-12-06T08:01:34,909 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false
2024-12-06T08:01:34,910 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 190 msec
2024-12-06T08:01:34,920 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:34,921 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4cbf4897 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e354cd0
2024-12-06T08:01:34,925 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:34,926 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137
2024-12-06T08:01:35,036 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:35,036 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:35,037 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:35,038 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:35,039 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139
2024-12-06T08:01:35,042 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:35,043 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x51c600f0 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@728bd183
2024-12-06T08:01:35,046 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x51c600f0 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x51c600f0 to 127.0.0.1:64333
2024-12-06T08:01:35,046 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:35,049 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139
2024-12-06T08:01:35,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=140
2024-12-06T08:01:35,201 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:35,202 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:35,202 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:35,202 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:35,203 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:35,204 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x07271cbd to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b03abeb
2024-12-06T08:01:35,204 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:35,247 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:35,247 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:35,248 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140
2024-12-06T08:01:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=140
2024-12-06T08:01:35,250 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:35,252 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139
2024-12-06T08:01:35,252 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 201 msec
2024-12-06T08:01:35,252 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:35,253 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 213 msec
2024-12-06T08:01:35,261 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:35,262 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a0dc73d to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@44e495c1
2024-12-06T08:01:35,265 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:35,265 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139
2024-12-06T08:01:35,356 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:35,356 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:35,357 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:35,358 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false
2024-12-06T08:01:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141
2024-12-06T08:01:35,361 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:35,362 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d0d4323 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@96b7387
2024-12-06T08:01:35,366 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x7d0d4323 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7d0d4323 to 127.0.0.1:64333
2024-12-06T08:01:35,366 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:35,370 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141
2024-12-06T08:01:35,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=142
2024-12-06T08:01:35,523 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:35,524 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:35,524 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:35,524 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:35,524 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:35,525 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x09c3734f to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2f56a8e5
2024-12-06T08:01:35,526 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:35,545 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:35,546 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:35,546 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142
2024-12-06T08:01:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=142
2024-12-06T08:01:35,548 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:35,550 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141
2024-12-06T08:01:35,550 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 178 msec
2024-12-06T08:01:35,550 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false
2024-12-06T08:01:35,552 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 192 msec
2024-12-06T08:01:35,560 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:35,562 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x217b633e to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5c0571d9
2024-12-06T08:01:35,565 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:35,565 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141
2024-12-06T08:01:35,676 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:35,676 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:35,677 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-12-06T08:01:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143
2024-12-06T08:01:35,679 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:35,680 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2d3c6adc to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5fa9efe9
2024-12-06T08:01:35,683 DEBUG [ReadOnlyZKClient-127.0.0.1:64333@0x2d3c6adc {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2d3c6adc to 127.0.0.1:64333
2024-12-06T08:01:35,684 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T08:01:35,688 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143
2024-12-06T08:01:35,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=144
2024-12-06T08:01:35,840 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-12-06T08:01:35,841 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-12-06T08:01:35,841 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-12-06T08:01:35,841 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:35,842 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:35,843 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0368cc72 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4130956e
2024-12-06T08:01:35,843 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:35,860 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(231): queueId=1-aa9080fe83d2,46375,1733472065852, ReplicationSource: 1, currentBandwidth=0
2024-12-06T08:01:35,860 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-12-06T08:01:35,860 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144
2024-12-06T08:01:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=144
2024-12-06T08:01:35,863 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:35,865 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143
2024-12-06T08:01:35,865 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 175 msec
2024-12-06T08:01:35,865 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64333/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-12-06T08:01:35,866 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 188 msec
2024-12-06T08:01:35,874 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:35,875 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a8fcdf0 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4c9827b5
2024-12-06T08:01:35,877 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 1 more
2024-12-06T08:01:35,878 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-12-06T08:01:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143
2024-12-06T08:01:35,995 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-12-06T08:01:35,996 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1
2024-12-06T08:01:35,996 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:35,996 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1
2024-12-06T08:01:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:01:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145
2024-12-06T08:01:36,000 DEBUG [PEWorker-4 {}] procedure.MasterProcedureScheduler(648): PEER '1', shared lock count=0, exclusively locked by procId=145
2024-12-06T08:01:36,000 DEBUG [PEWorker-4 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=146, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-12-06T08:01:36,000 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:36,000 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:36,001 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:36,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-12-06T08:01:36,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:36,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-12-06T08:01:36,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-12-06T08:01:36,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39057-0x10186ca7dde0000, quorum=127.0.0.1:64333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-12-06T08:01:36,010 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-12-06T08:01:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:01:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145
2024-12-06T08:01:36,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=147
2024-12-06T08:01:36,162 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-12-06T08:01:36,163 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-12-06T08:01:36,163 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-aa9080fe83d2,46375,1733472065852 because: Replication stream was removed by a user
2024-12-06T08:01:36,163 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-12-06T08:01:36,163 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64333' and parent='/hbase-test1'
2024-12-06T08:01:36,164 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6c34dc51 to 127.0.0.1:64333 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2f3b6e03
2024-12-06T08:01:36,164 WARN  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0.replicationSource,1-aa9080fe83d2,46375,1733472065852 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 7 more
2024-12-06T08:01:36,165 INFO  [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSourceManager(562): Done with the queue 1-aa9080fe83d2,46375,1733472065852
2024-12-06T08:01:36,167 DEBUG [RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147
2024-12-06T08:01:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.HMaster(4169): Remote procedure done, pid=147
2024-12-06T08:01:36,169 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on aa9080fe83d2,46375,1733472065852 suceeded
2024-12-06T08:01:36,170 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145
2024-12-06T08:01:36,170 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec
2024-12-06T08:01:36,170 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-12-06T08:01:36,170 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-12-06T08:01:36,175 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-12-06T08:01:36,176 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 179 msec
2024-12-06T08:01:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:01:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145
2024-12-06T08:01:36,316 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-12-06T08:01:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:01:37,119 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-06T08:01:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:01:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:01:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:01:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:01:49,295 INFO  [master/aa9080fe83d2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-06T08:01:49,295 INFO  [master/aa9080fe83d2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-06T08:01:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:02:04,264 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:02:04,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:02:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:02:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:02:34,265 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:02:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:03:04,265 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:03:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:03:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:03:34,265 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:03:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:04:04,265 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:04:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:04:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:04:34,266 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:04:45,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:05:04,266 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:05:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:05:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:05:34,266 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:05:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:06:04,266 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:06:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:06:05,930 INFO  [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=660.99 KB, freeSize=879.35 MB, max=880 MB, blockCount=0, accesses=0, hits=0, hitRatio=0, cachingAccesses=0, cachingHits=0, cachingHitsRatio=0,evictions=29, evicted=0, evictedPerRun=0.0
2024-12-06T08:06:05,993 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=0, created chunk count=9, reused chunk count=0, reuseRatio=0.00%
2024-12-06T08:06:05,994 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-12-06T08:06:08,290 INFO  [regionserver/aa9080fe83d2:0.Chore.1 {}] regionserver.Replication$ReplicationStatisticsChore(208): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B

2024-12-06T08:06:09,202 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T08:06:09,204 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50167, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T08:06:09,210 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T08:06:09,222 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default
2024-12-06T08:06:09,222 INFO  [master/aa9080fe83d2:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default
2024-12-06T08:06:09,222 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster.
2024-12-06T08:06:09,225 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(204): Hosts are {aa9080fe83d2=0} racks are {/default-rack=0}
2024-12-06T08:06:09,232 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(303): server 0 has 2 regions
2024-12-06T08:06:09,232 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-12-06T08:06:09,232 INFO  [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-12-06T08:06:09,232 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1
2024-12-06T08:06:09,234 INFO  [master/aa9080fe83d2:0.Chore.1 {}] balancer.StochasticLoadBalancer(357): Not running balancer because only 1 active regionserver(s)
2024-12-06T08:06:09,235 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms
2024-12-06T08:06:09,265 INFO  [regionserver/aa9080fe83d2:0.Chore.1 {}] regionserver.HRegionServer$PeriodicMemStoreFlusher(1763): MemstoreFlusherChore requesting flush of hbase:meta,,1.1588230740 because 1588230740/ns has an old edit so flush to free WALs after random delay 86449 ms
2024-12-06T08:06:11,269 INFO  [regionserver/aa9080fe83d2:0.Chore.1 {}] regionserver.HRegionServer$PeriodicMemStoreFlusher(1763): MemstoreFlusherChore requesting flush of hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea. because 5f2f3593e74f6cff3d351f05bde952ea/queue has an old edit so flush to free WALs after random delay 41562 ms
2024-12-06T08:06:14,487 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-06T08:06:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:06:34,267 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:06:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:06:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-12-06T08:06:52,835 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f2f3593e74f6cff3d351f05bde952ea 3/3 column families, dataSize=2.75 KB heapSize=6.19 KB
2024-12-06T08:06:52,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/replication/5f2f3593e74f6cff3d351f05bde952ea/.tmp/hfileref/ca0a1124cc874a31a351c64192a39f40 is 33, key is 1/hfileref:/1733472096174/DeleteFamily/seqid=0
2024-12-06T08:06:52,913 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741838_1014 (size=5433)
2024-12-06T08:06:53,314 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=348 B at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/replication/5f2f3593e74f6cff3d351f05bde952ea/.tmp/hfileref/ca0a1124cc874a31a351c64192a39f40
2024-12-06T08:06:53,348 INFO  [MemStoreFlusher.0 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ca0a1124cc874a31a351c64192a39f40
2024-12-06T08:06:53,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/replication/5f2f3593e74f6cff3d351f05bde952ea/.tmp/queue/35042319b89845b58212591954c0ca97 is 63, key is 1-aa9080fe83d2,46375,1733472065852/queue:/1733472096166/DeleteFamily/seqid=0
2024-12-06T08:06:53,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741839_1015 (size=5982)
2024-12-06T08:06:53,785 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.41 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/replication/5f2f3593e74f6cff3d351f05bde952ea/.tmp/queue/35042319b89845b58212591954c0ca97
2024-12-06T08:06:53,793 INFO  [MemStoreFlusher.0 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 35042319b89845b58212591954c0ca97
2024-12-06T08:06:53,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/replication/5f2f3593e74f6cff3d351f05bde952ea/.tmp/hfileref/ca0a1124cc874a31a351c64192a39f40 as hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/replication/5f2f3593e74f6cff3d351f05bde952ea/hfileref/ca0a1124cc874a31a351c64192a39f40
2024-12-06T08:06:53,802 INFO  [MemStoreFlusher.0 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ca0a1124cc874a31a351c64192a39f40
2024-12-06T08:06:53,802 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/replication/5f2f3593e74f6cff3d351f05bde952ea/hfileref/ca0a1124cc874a31a351c64192a39f40, entries=12, sequenceid=40, filesize=5.3 K
2024-12-06T08:06:53,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/replication/5f2f3593e74f6cff3d351f05bde952ea/.tmp/queue/35042319b89845b58212591954c0ca97 as hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/replication/5f2f3593e74f6cff3d351f05bde952ea/queue/35042319b89845b58212591954c0ca97
2024-12-06T08:06:53,812 INFO  [MemStoreFlusher.0 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 35042319b89845b58212591954c0ca97
2024-12-06T08:06:53,812 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/replication/5f2f3593e74f6cff3d351f05bde952ea/queue/35042319b89845b58212591954c0ca97, entries=12, sequenceid=40, filesize=5.8 K
2024-12-06T08:06:53,817 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.75 KB/2820, heapSize ~5.91 KB/6048, currentSize=0 B/0 for 5f2f3593e74f6cff3d351f05bde952ea in 981ms, sequenceid=40, compaction requested=false
2024-12-06T08:06:53,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f2f3593e74f6cff3d351f05bde952ea:

2024-12-06T08:07:04,267 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:07:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:07:08,279 WARN  [regionserver/aa9080fe83d2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1
2024-12-06T08:07:10,624 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:07:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:07:34,267 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:07:35,714 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-12-06T08:07:35,715 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.34 KB heapSize=3.38 KB
2024-12-06T08:07:35,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/.tmp/info/9b126c869b4641cbae1b8ae83d1495a6 is 147, key is hbase:replication,,1733472069413.5f2f3593e74f6cff3d351f05bde952ea./info:regioninfo/1733472070655/Put/seqid=0
2024-12-06T08:07:35,757 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741840_1016 (size=6631)
2024-12-06T08:07:36,158 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.17 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/.tmp/info/9b126c869b4641cbae1b8ae83d1495a6
2024-12-06T08:07:36,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/.tmp/ns/6f95dcd7ca774c5e90bdf26a8f667d51 is 43, key is default/ns:d/1733472069102/Put/seqid=0
2024-12-06T08:07:36,187 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741841_1017 (size=5153)
2024-12-06T08:07:36,589 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/.tmp/ns/6f95dcd7ca774c5e90bdf26a8f667d51
2024-12-06T08:07:36,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/.tmp/table/e64da09009da4415adfc603e9f3647db is 53, key is hbase:replication/table:state/1733472070671/Put/seqid=0
2024-12-06T08:07:36,616 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741842_1018 (size=5256)
2024-12-06T08:07:37,018 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=98 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/.tmp/table/e64da09009da4415adfc603e9f3647db
2024-12-06T08:07:37,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/.tmp/info/9b126c869b4641cbae1b8ae83d1495a6 as hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/info/9b126c869b4641cbae1b8ae83d1495a6
2024-12-06T08:07:37,035 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/info/9b126c869b4641cbae1b8ae83d1495a6, entries=10, sequenceid=11, filesize=6.5 K
2024-12-06T08:07:37,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/.tmp/ns/6f95dcd7ca774c5e90bdf26a8f667d51 as hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/ns/6f95dcd7ca774c5e90bdf26a8f667d51
2024-12-06T08:07:37,043 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/ns/6f95dcd7ca774c5e90bdf26a8f667d51, entries=2, sequenceid=11, filesize=5.0 K
2024-12-06T08:07:37,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/.tmp/table/e64da09009da4415adfc603e9f3647db as hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/table/e64da09009da4415adfc603e9f3647db
2024-12-06T08:07:37,052 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/data/hbase/meta/1588230740/table/e64da09009da4415adfc603e9f3647db, entries=2, sequenceid=11, filesize=5.1 K
2024-12-06T08:07:37,053 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.34 KB/1368, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 1338ms, sequenceid=11, compaction requested=false
2024-12-06T08:07:37,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-06T08:07:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:07:53,969 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:07:55,625 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:08:04,267 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:08:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:08:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:08:34,268 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:08:38,969 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:08:40,625 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:08:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:09:04,268 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:09:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:09:23,969 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:09:25,625 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:09:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:09:34,268 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:09:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:10:04,268 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:10:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:10:08,970 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:10:10,626 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:10:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:10:34,269 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:10:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:10:53,970 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:10:55,626 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:11:04,269 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:11:05,925 INFO  [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=660.99 KB, freeSize=879.35 MB, max=880 MB, blockCount=0, accesses=0, hits=0, hitRatio=0, cachingAccesses=0, cachingHits=0, cachingHitsRatio=0,evictions=59, evicted=0, evictedPerRun=0.0
2024-12-06T08:11:05,993 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=5, created chunk count=9, reused chunk count=0, reuseRatio=0.00%
2024-12-06T08:11:05,994 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-12-06T08:11:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:11:08,060 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T08:11:08,062 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55689, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T08:11:08,309 WARN  [regionserver/aa9080fe83d2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 4, running: 0
2024-12-06T08:11:08,309 INFO  [regionserver/aa9080fe83d2:0.Chore.1 {}] regionserver.Replication$ReplicationStatisticsChore(208): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B

2024-12-06T08:11:09,259 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T08:11:09,260 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default
2024-12-06T08:11:09,260 INFO  [master/aa9080fe83d2:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default
2024-12-06T08:11:09,260 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster.
2024-12-06T08:11:09,261 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(204): Hosts are {aa9080fe83d2=0} racks are {/default-rack=0}
2024-12-06T08:11:09,261 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(303): server 0 has 2 regions
2024-12-06T08:11:09,261 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-12-06T08:11:09,261 INFO  [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-12-06T08:11:09,261 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1
2024-12-06T08:11:09,261 INFO  [master/aa9080fe83d2:0.Chore.1 {}] balancer.StochasticLoadBalancer(357): Not running balancer because only 1 active regionserver(s)
2024-12-06T08:11:09,261 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms
2024-12-06T08:11:14,483 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-06T08:11:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:11:34,269 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:11:38,970 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:11:40,626 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:11:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:12:04,269 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:12:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:12:23,970 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:12:25,626 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:12:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:12:34,269 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:12:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:13:04,270 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:13:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:13:08,971 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:13:10,627 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:13:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:13:34,270 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:13:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:13:53,971 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:13:55,627 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:14:04,270 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:14:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:14:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:14:34,270 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:14:38,971 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:14:40,627 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:14:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:15:04,271 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:15:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:15:08,343 WARN  [regionserver/aa9080fe83d2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1
2024-12-06T08:15:23,972 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:15:25,628 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:15:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:15:34,271 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:15:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:16:04,271 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:16:05,925 INFO  [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=665.45 KB, freeSize=879.35 MB, max=880 MB, blockCount=4, accesses=5, hits=1, hitRatio=20.00%, , cachingAccesses=5, cachingHits=1, cachingHitsRatio=20.00%, evictions=89, evicted=0, evictedPerRun=0.0
2024-12-06T08:16:05,993 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=5, created chunk count=9, reused chunk count=0, reuseRatio=0.00%
2024-12-06T08:16:05,994 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-12-06T08:16:07,592 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(201): WAL AsyncFSWAL aa9080fe83d2%2C39057%2C1733472065171:(num 1733472067512) roll period 900000 ms elapsed
2024-12-06T08:16:07,606 DEBUG [master:store-WAL-Roller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/WALs/aa9080fe83d2,39057,1733472065171/aa9080fe83d2%2C39057%2C1733472065171.1733472967593, exclude list is [], retry=0
2024-12-06T08:16:07,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32829,DS-2734aae5-af90-40e7-b9d8-6e9731ae49a4,DISK]
2024-12-06T08:16:07,612 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/WALs/aa9080fe83d2,39057,1733472065171/aa9080fe83d2%2C39057%2C1733472065171.1733472067512 with entries=1148, filesize=511.42 KB; new WAL /user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/WALs/aa9080fe83d2,39057,1733472065171/aa9080fe83d2%2C39057%2C1733472065171.1733472967593
2024-12-06T08:16:07,613 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39251:39251)]
2024-12-06T08:16:07,613 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/WALs/aa9080fe83d2,39057,1733472065171/aa9080fe83d2%2C39057%2C1733472065171.1733472067512 is not closed yet, will try archiving it next time
2024-12-06T08:16:07,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741830_1006 (size=523704)
2024-12-06T08:16:07,708 INFO  [master:store-Flusher {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=437.79 KB heapSize=538.20 KB
2024-12-06T08:16:07,726 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa507a449a1a477e90b3cd201fbf8ee7 is 82, key is hbase:meta,,1/info:regioninfo/1733472069023/Put/seqid=0
2024-12-06T08:16:07,730 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741844_1020 (size=5672)
2024-12-06T08:16:08,138 INFO  [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1151 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa507a449a1a477e90b3cd201fbf8ee7
2024-12-06T08:16:08,165 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/54c850f79b5a42eeb56670be6157bfae is 293, key is \x00\x00\x00\x00\x00\x00\x00\x92/proc:d/1733472095997/Put/seqid=0
2024-12-06T08:16:08,169 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741845_1021 (size=11215)
2024-12-06T08:16:08,315 INFO  [regionserver/aa9080fe83d2:0.Chore.1 {}] regionserver.Replication$ReplicationStatisticsChore(208): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B

2024-12-06T08:16:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:16:08,571 INFO  [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=436.24 KB at sequenceid=1151 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/54c850f79b5a42eeb56670be6157bfae
2024-12-06T08:16:08,577 INFO  [master:store-Flusher {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 54c850f79b5a42eeb56670be6157bfae
2024-12-06T08:16:08,592 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fe34f56fcd944760919c71bd4853c076 is 69, key is aa9080fe83d2,46375,1733472065852/rs:state/1733472068187/Put/seqid=0
2024-12-06T08:16:08,597 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741846_1022 (size=5156)
2024-12-06T08:16:08,972 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:16:08,998 INFO  [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1151 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fe34f56fcd944760919c71bd4853c076
2024-12-06T08:16:09,022 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0b8f0d8ec3f14fd69dd45a5155b0c6a9 is 68, key is replication_peer_modification_on/state:d/1733472093729/Put/seqid=0
2024-12-06T08:16:09,027 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32829 is added to blk_1073741847_1023 (size=5154)
2024-12-06T08:16:09,151 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T08:16:09,153 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36787, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T08:16:09,156 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T08:16:09,158 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default
2024-12-06T08:16:09,158 INFO  [master/aa9080fe83d2:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default
2024-12-06T08:16:09,158 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster.
2024-12-06T08:16:09,158 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(204): Hosts are {aa9080fe83d2=0} racks are {/default-rack=0}
2024-12-06T08:16:09,158 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(303): server 0 has 2 regions
2024-12-06T08:16:09,158 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-12-06T08:16:09,158 INFO  [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-12-06T08:16:09,158 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1
2024-12-06T08:16:09,159 INFO  [master/aa9080fe83d2:0.Chore.1 {}] balancer.StochasticLoadBalancer(357): Not running balancer because only 1 active regionserver(s)
2024-12-06T08:16:09,159 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms
2024-12-06T08:16:09,428 INFO  [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1 KB at sequenceid=1151 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0b8f0d8ec3f14fd69dd45a5155b0c6a9
2024-12-06T08:16:09,436 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa507a449a1a477e90b3cd201fbf8ee7 as hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aa507a449a1a477e90b3cd201fbf8ee7
2024-12-06T08:16:09,443 INFO  [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aa507a449a1a477e90b3cd201fbf8ee7, entries=8, sequenceid=1151, filesize=5.5 K
2024-12-06T08:16:09,444 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/54c850f79b5a42eeb56670be6157bfae as hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/54c850f79b5a42eeb56670be6157bfae
2024-12-06T08:16:09,451 INFO  [master:store-Flusher {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 54c850f79b5a42eeb56670be6157bfae
2024-12-06T08:16:09,451 INFO  [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/54c850f79b5a42eeb56670be6157bfae, entries=147, sequenceid=1151, filesize=11.0 K
2024-12-06T08:16:09,452 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fe34f56fcd944760919c71bd4853c076 as hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fe34f56fcd944760919c71bd4853c076
2024-12-06T08:16:09,459 INFO  [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fe34f56fcd944760919c71bd4853c076, entries=1, sequenceid=1151, filesize=5.0 K
2024-12-06T08:16:09,460 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0b8f0d8ec3f14fd69dd45a5155b0c6a9 as hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0b8f0d8ec3f14fd69dd45a5155b0c6a9
2024-12-06T08:16:09,465 INFO  [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0b8f0d8ec3f14fd69dd45a5155b0c6a9, entries=1, sequenceid=1151, filesize=5.0 K
2024-12-06T08:16:09,466 INFO  [master:store-Flusher {}] regionserver.HRegion(3140): Finished flush of dataSize ~437.79 KB/448300, heapSize ~538.14 KB/551056, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1758ms, sequenceid=1151, compaction requested=false
2024-12-06T08:16:09,466 DEBUG [master:store-Flusher {}] regionserver.HRegion(2603): Flush status journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-06T08:16:10,628 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:16:14,486 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-06T08:16:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:16:34,271 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:16:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:16:53,972 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:16:55,628 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:17:04,271 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:17:08,328 WARN  [regionserver/aa9080fe83d2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 4, running: 1
2024-12-06T08:17:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:17:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:17:34,272 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:17:38,973 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:17:40,629 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:17:48,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:18:04,272 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:18:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:18:23,973 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:18:25,629 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:18:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:18:34,272 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:18:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:19:04,272 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:19:08,363 WARN  [regionserver/aa9080fe83d2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1
2024-12-06T08:19:08,973 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:19:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:19:10,629 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:19:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:19:34,272 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:19:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:19:53,973 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:19:55,629 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:20:04,273 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:20:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:20:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:20:34,273 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:20:38,974 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:20:40,630 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:20:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:21:04,273 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:21:05,925 INFO  [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=665.45 KB, freeSize=879.35 MB, max=880 MB, blockCount=4, accesses=7, hits=3, hitRatio=42.86%, , cachingAccesses=7, cachingHits=3, cachingHitsRatio=42.86%, evictions=119, evicted=0, evictedPerRun=0.0
2024-12-06T08:21:05,993 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=8, created chunk count=9, reused chunk count=1, reuseRatio=10.00%
2024-12-06T08:21:05,994 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-12-06T08:21:08,264 WARN  [regionserver/aa9080fe83d2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 1, running: 1
2024-12-06T08:21:08,291 INFO  [regionserver/aa9080fe83d2:0.Chore.1 {}] regionserver.Replication$ReplicationStatisticsChore(208): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B

2024-12-06T08:21:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:21:13,770 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T08:21:13,771 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45949, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T08:21:13,774 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T08:21:13,791 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default
2024-12-06T08:21:13,791 INFO  [master/aa9080fe83d2:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default
2024-12-06T08:21:13,791 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster.
2024-12-06T08:21:13,791 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(204): Hosts are {aa9080fe83d2=0} racks are {/default-rack=0}
2024-12-06T08:21:13,791 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(303): server 0 has 2 regions
2024-12-06T08:21:13,791 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-12-06T08:21:13,791 INFO  [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-12-06T08:21:13,791 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1
2024-12-06T08:21:13,792 INFO  [master/aa9080fe83d2:0.Chore.1 {}] balancer.StochasticLoadBalancer(357): Not running balancer because only 1 active regionserver(s)
2024-12-06T08:21:13,792 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms
2024-12-06T08:21:14,483 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-06T08:21:23,974 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:21:25,630 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:21:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:21:34,273 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:21:50,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:22:04,273 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:22:08,974 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:22:10,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:22:10,630 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:22:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:22:34,274 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:22:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:22:53,975 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:22:55,631 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:23:04,274 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:23:08,317 WARN  [regionserver/aa9080fe83d2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 5, running: 1
2024-12-06T08:23:10,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:23:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:23:34,274 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:23:38,975 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:23:40,631 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:23:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:24:04,274 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:24:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:24:23,975 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:24:25,631 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:24:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:24:34,275 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:24:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:25:04,275 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:25:08,976 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:25:10,631 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:25:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:25:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:25:34,275 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:25:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:25:53,976 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:25:55,632 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:26:04,275 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:26:05,925 INFO  [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=665.45 KB, freeSize=879.35 MB, max=880 MB, blockCount=4, accesses=12, hits=8, hitRatio=66.67%, , cachingAccesses=12, cachingHits=8, cachingHitsRatio=66.67%, evictions=149, evicted=0, evictedPerRun=0.0
2024-12-06T08:26:05,993 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=8, created chunk count=9, reused chunk count=1, reuseRatio=10.00%
2024-12-06T08:26:05,994 DEBUG [master/aa9080fe83d2:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-12-06T08:26:08,291 INFO  [regionserver/aa9080fe83d2:0.Chore.1 {}] regionserver.Replication$ReplicationStatisticsChore(208): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B

2024-12-06T08:26:09,150 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default
2024-12-06T08:26:09,150 INFO  [master/aa9080fe83d2:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default
2024-12-06T08:26:09,150 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster.
2024-12-06T08:26:09,150 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(204): Hosts are {aa9080fe83d2=0} racks are {/default-rack=0}
2024-12-06T08:26:09,150 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(303): server 0 has 2 regions
2024-12-06T08:26:09,150 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-12-06T08:26:09,150 INFO  [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-12-06T08:26:09,150 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1
2024-12-06T08:26:09,150 INFO  [master/aa9080fe83d2:0.Chore.1 {}] balancer.StochasticLoadBalancer(357): Not running balancer because only 1 active regionserver(s)
2024-12-06T08:26:09,150 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms
2024-12-06T08:26:09,151 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T08:26:09,152 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36655, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T08:26:09,156 DEBUG [master/aa9080fe83d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T08:26:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:26:14,481 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-06T08:26:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-12-06T08:26:34,275 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T08:26:38,976 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040
2024-12-06T08:26:40,632 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5f2f3593e74f6cff3d351f05bde952ea, had cached 0 bytes from a total of 11415
2024-12-06T08:26:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
====> TEST TIMED OUT. PRINTING THREAD DUMP. <====

Timestamp: 2024-12-06 08:27:01,609

"NIOWorkerThread-14" daemon prio=5 tid=170 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-5" daemon prio=5 tid=161 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"AsyncFSWAL-0-hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2-prefix:aa9080fe83d2,46375,1733472065852" daemon prio=5 tid=289 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x3450c59c" daemon prio=5 tid=476 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x6d9982c2" daemon prio=5 tid=482 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x59a08698" daemon prio=5 tid=653 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Timer for 'HBase' metrics system" daemon prio=5 tid=1001 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
        at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
"qtp1269359400-37" daemon prio=5 tid=37 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f6f6842a550.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp1269359400-39" daemon prio=5 tid=39 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f6f6842a550.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"normalizer-worker-0" daemon prio=5 tid=254 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at app//org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorkQueue.take(RegionNormalizerWorkQueue.java:146)
        at app//org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorker.run(RegionNormalizerWorker.java:191)
        at java.base@17.0.11/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Command processor" daemon prio=5 tid=101 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395)
"RegionServerTracker-0" daemon prio=5 tid=274 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RS_REFRESH_PEER-regionserver/aa9080fe83d2:0-0" daemon prio=5 tid=333 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-7" daemon prio=5 tid=163 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"master/aa9080fe83d2:0:becomeActiveMaster-MemStoreChunkPool Statistics" daemon prio=5 tid=214 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x6e9cf834" daemon prio=5 tid=547 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"LruBlockCacheStatsExecutor" daemon prio=5 tid=194 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-15" daemon prio=5 tid=171 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp1269359400-38" daemon prio=5 tid=38 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f6f6842a550.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x1779283c" daemon prio=5 tid=345 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x07271cbd" daemon prio=5 tid=707 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39057" daemon prio=5 tid=178 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"DataXceiver for client DFSClient_NONMAPREDUCE_820960735_22 at /127.0.0.1:54240 [Receiving block BP-367880231-172.17.0.2-1733472062403:blk_1073741837_1013]" daemon prio=5 tid=326 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
        at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
        at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
        at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
        at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
        at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151)
        at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x21092368" daemon prio=5 tid=427 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-8" daemon prio=5 tid=164 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39057" daemon prio=5 tid=177 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"ReadOnlyZKClient-127.0.0.1:64333@0x78c94197" daemon prio=5 tid=385 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x7eb9a8d7" daemon prio=5 tid=453 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"MobFileCache #0" daemon prio=5 tid=196 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"master:store-Flusher" daemon prio=5 tid=241 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759)
        at app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor.flushLoop(MasterRegionFlusherAndCompactor.java:200)
        at app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor$$Lambda$481/0x00007f6f68a199c0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.util.JvmPauseMonitor$Monitor@42b50692" daemon prio=5 tid=34 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"MarkedDeleteBlockScrubberThread" daemon prio=5 tid=48 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$MarkedDeleteBlockScrubber.run(BlockManager.java:5326)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"zk-event-processor-pool-0" daemon prio=5 tid=160 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"surefire-forkedjvm-command-thread" daemon prio=5 tid=18 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at java.base@17.0.11/sun.nio.ch.PendingFuture.get(PendingFuture.java:178)
        at app//org.apache.maven.surefire.api.util.internal.Channels$2.read(Channels.java:127)
        at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
        at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
        at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
        at app//org.apache.maven.surefire.api.util.internal.Channels$3.readImpl(Channels.java:169)
        at app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleReadableChannel.read(AbstractNoninterruptibleReadableChannel.java:50)
        at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.read(AbstractStreamDecoder.java:430)
        at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.read(AbstractStreamDecoder.java:419)
        at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.readMessageType(AbstractStreamDecoder.java:116)
        at app//org.apache.maven.surefire.booter.stream.CommandDecoder.decode(CommandDecoder.java:77)
        at app//org.apache.maven.surefire.booter.spi.CommandChannelDecoder.decode(CommandChannelDecoder.java:60)
        at app//org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:290)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"DataXceiver for client DFSClient_NONMAPREDUCE_-342368316_22 at /127.0.0.1:39816 [Receiving block BP-367880231-172.17.0.2-1733472062403:blk_1073741843_1019]" daemon prio=5 tid=888 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
        at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
        at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
        at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
        at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
        at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151)
        at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x73c9735c" daemon prio=5 tid=627 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"HBase-Metrics2-1" daemon prio=5 tid=153 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x1ef65269" daemon prio=5 tid=381 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"pool-17-thread-1"  prio=5 tid=86 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Socket Reader #1 for port 0" daemon prio=5 tid=95 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497)
        at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476)
"Session-HouseKeeper-7a835e3c-1"  prio=5 tid=45 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server listener on 0" daemon prio=5 tid=94 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559)
"ReadOnlyZKClient-127.0.0.1:64333@0x65277045" daemon prio=5 tid=610 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x4251adba" daemon prio=5 tid=509 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"nioEventLoopGroup-2-1"  prio=10 tid=92 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
        at app//io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
        at app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
        at app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
        at app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Registry-endpoints-refresh-end-points" daemon prio=5 tid=307 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hbase.client.RegistryEndpointsRefresher.mainLoop(RegistryEndpointsRefresher.java:92)
        at app//org.apache.hadoop.hbase.client.RegistryEndpointsRefresher$$Lambda$806/0x00007f6f68bb1750.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x66015df2" daemon prio=5 tid=341 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100" daemon prio=5 tid=35 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)
"ReadOnlyZKClient-127.0.0.1:64333@0x4cbf4897" daemon prio=5 tid=701 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=46375" daemon prio=5 tid=205 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"SessionTracker" daemon prio=5 tid=141 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)
"java.util.concurrent.ThreadPoolExecutor$Worker@332536fb[State = -1, empty queue]" daemon prio=5 tid=135 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x2c7d4e25" daemon prio=5 tid=391 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server handler 1 on default port 34433" daemon prio=5 tid=65 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"ReadOnlyZKClient-127.0.0.1:64333@0x0df8c4c3" daemon prio=5 tid=613 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"FSEditLogAsync" daemon prio=5 tid=53 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
        at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:241)
        at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:250)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46375" daemon prio=5 tid=199 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"ReadOnlyZKClient-127.0.0.1:64333@0x479b6aff" daemon prio=5 tid=567 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x6696e4e2" daemon prio=5 tid=355 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"SnapshotHandlerChoreCleaner" daemon prio=5 tid=255 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Async-Client-Retry-Timer-pool-0" daemon prio=5 tid=242 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
        at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ConnnectionExpirer" daemon prio=5 tid=137 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$ConnectionExpirerThread.run(NIOServerCnxnFactory.java:554)
"PEWorker-3" daemon prio=5 tid=249 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149)
        at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176)
"PEWorker-1" daemon prio=5 tid=247 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149)
        at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176)
"ReadOnlyZKClient-127.0.0.1:64333@0x4b49370a" daemon prio=5 tid=691 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x4a62a85e" daemon prio=5 tid=586 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.util.JvmPauseMonitor$Monitor@7b2fe16" daemon prio=5 tid=93 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x212a817d" daemon prio=5 tid=351 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server handler 0 on default port 34433" daemon prio=5 tid=64 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39057" daemon prio=5 tid=174 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=46375" daemon prio=5 tid=203 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"NIOWorkerThread-2" daemon prio=5 tid=157 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x09c3734f" daemon prio=5 tid=717 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ProcedureDispatcherTimeoutThread" daemon prio=5 tid=252 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81)
        at app//org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher$TimeoutExecutorThread.run(RemoteProcedureDispatcher.java:328)
"master/aa9080fe83d2:0:becomeActiveMaster-MemStoreChunkPool Statistics" daemon prio=5 tid=212 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RPCClient-NioEventLoopGroup-4-4" daemon prio=5 tid=308 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"MiniHBaseClusterRegionServer-EventLoopGroup-3-3" daemon prio=10 tid=294 runnable
java.lang.Thread.State: RUNNABLE
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Signal Dispatcher" daemon prio=9 tid=4 runnable
java.lang.Thread.State: RUNNABLE
"RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=39057" daemon prio=5 tid=180 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"ReadOnlyZKClient-127.0.0.1:64333@0x6401d8d4" daemon prio=5 tid=557 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=46375" daemon prio=5 tid=206 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"ReadOnlyZKClient-127.0.0.1:64333@0x56ff058f" daemon prio=5 tid=526 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp878742499-87" daemon prio=5 tid=87 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f6f6842a550.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39057" daemon prio=5 tid=175 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"ReadOnlyZKClient-127.0.0.1:64333@0x399e41b7" daemon prio=5 tid=543 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x67e378ba" daemon prio=5 tid=679 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Socket Reader #1 for port 0" daemon prio=5 tid=55 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497)
        at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476)
"ReadOnlyZKClient-127.0.0.1:64333@0x69df80da" daemon prio=5 tid=596 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x03362617" daemon prio=5 tid=496 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server Responder" daemon prio=5 tid=97 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733)
        at app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716)
"Notification Thread" daemon prio=9 tid=13 runnable
java.lang.Thread.State: RUNNABLE
"RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46375" daemon prio=5 tid=198 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"pool-6-thread-1"  prio=5 tid=46 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp878742499-88-acceptor-0@51811433-ServerConnector@43176244{HTTP/1.1, (http/1.1)}{localhost:40603}" daemon prio=3 tid=88 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method)
        at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
        at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RequestThrottler" daemon prio=5 tid=144 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
"ReadOnlyZKClient-127.0.0.1:64333@0x1381c835" daemon prio=5 tid=371 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"pool-1-thread-2" daemon prio=5 tid=15 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.poll(EPollPort.java:200)
        at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.run(EPollPort.java:281)
        at java.base@17.0.11/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:113)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Time-limited test-SendThread(127.0.0.1:64333)" daemon prio=5 tid=189 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
"RS:0;aa9080fe83d2:46375-longCompactions-0" daemon prio=5 tid=278 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at app//org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:101)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"LeaseRenewer:jenkins@localhost:34433" daemon prio=5 tid=147 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
        at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
        at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x1de25d92" daemon prio=5 tid=413 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Block report processor" daemon prio=5 tid=51 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.processQueue(BlockManager.java:5627)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.run(BlockManager.java:5614)
"HMaster-EventLoopGroup-1-1" daemon prio=10 tid=154 runnable
java.lang.Thread.State: RUNNABLE
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp1269359400-42" daemon prio=5 tid=42 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x294e719b" daemon prio=5 tid=395 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Reference Handler" daemon prio=10 tid=2 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/java.lang.ref.Reference.waitForReferencePendingList(Native Method)
        at java.base@17.0.11/java.lang.ref.Reference.processPendingReferences(Reference.java:253)
        at java.base@17.0.11/java.lang.ref.Reference$ReferenceHandler.run(Reference.java:215)
"ReadOnlyZKClient-127.0.0.1:64333@0x739ac608" daemon prio=5 tid=486 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"DatanodeAdminMonitor-0" daemon prio=5 tid=62 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/cluster_acdf4e60-e0ee-bbdc-8ae2-647dd3ea8ddc/data/data2)" daemon prio=5 tid=118 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656)
"IPC Server handler 1 on default port 38083" daemon prio=5 tid=105 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=46375" daemon prio=5 tid=204 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"NIOWorkerThread-6" daemon prio=5 tid=162 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x22d5fed8" daemon prio=5 tid=406 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x790fd412" daemon prio=5 tid=469 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x6c34dc51" daemon prio=5 tid=734 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server handler 2 on default port 38083" daemon prio=5 tid=106 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"HMaster-EventLoopGroup-1-3" daemon prio=10 tid=304 runnable
java.lang.Thread.State: RUNNABLE
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x5fc5b243" daemon prio=5 tid=669 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Time-limited test-EventThread" daemon prio=5 tid=156 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
"NIOWorkerThread-13" daemon prio=5 tid=169 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x6ea69fe7" daemon prio=5 tid=466 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"regionserver/aa9080fe83d2:0.logRoller" daemon prio=5 tid=281 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180)
"VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/cluster_acdf4e60-e0ee-bbdc-8ae2-647dd3ea8ddc/data/data1)" daemon prio=5 tid=117 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656)
"RS:0;aa9080fe83d2:46375" daemon prio=5 tid=207 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:81)
        at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:64)
        at app//org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:906)
        at app//org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
        at app//org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
        at java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776)
        at java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:399)
        at java.base@17.0.11/javax.security.auth.Subject.doAs(Subject.java:376)
        at app//org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
        at app//org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
        at app//org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"SSL Certificates Store Monitor" daemon prio=5 tid=25 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.Object.wait(Object.java:338)
        at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
        at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
"RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46375" daemon prio=5 tid=202 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"RPCClient-NioEventLoopGroup-4-6" daemon prio=5 tid=310 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@6b954844" daemon prio=5 tid=71 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:537)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Idle-Rpc-Conn-Sweeper-pool-0" daemon prio=5 tid=244 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x0368cc72" daemon prio=5 tid=727 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-4" daemon prio=5 tid=159 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x2b5480af" daemon prio=5 tid=506 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp878742499-89" daemon prio=5 tid=89 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"BP-367880231-172.17.0.2-1733472062403 heartbeating to localhost/127.0.0.1:34433" daemon prio=5 tid=102 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:771)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:914)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/cluster_acdf4e60-e0ee-bbdc-8ae2-647dd3ea8ddc/data/data1/current/BP-367880231-172.17.0.2-1733472062403" daemon prio=5 tid=123 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"zk-event-processor-pool-0" daemon prio=5 tid=191 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"aa9080fe83d2:39057" daemon prio=5 tid=253 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager.waitOnAssignQueue(AssignmentManager.java:2398)
        at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager.processAssignQueue(AssignmentManager.java:2420)
        at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager$1.run(AssignmentManager.java:2360)
"Hadoop-Metrics-Updater-0" daemon prio=5 tid=58 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Client (1792563440) connection to localhost/127.0.0.1:34433 from jenkins" daemon prio=5 tid=109 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
        at app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
"OldWALsCleaner-0" daemon prio=5 tid=261 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner.deleteFile(LogCleaner.java:172)
        at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner.lambda$createOldWalsCleaner$1(LogCleaner.java:152)
        at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner$$Lambda$611/0x00007f6f68af61f0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"PacketResponder: BP-367880231-172.17.0.2-1733472062403:blk_1073741834_1010, type=LAST_IN_PIPELINE" daemon prio=5 tid=297 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.Object.wait(Object.java:338)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server listener on 0" daemon prio=5 tid=54 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559)
"FsDatasetAsyncDiskServiceFixer" daemon prio=5 tid=136 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer.run(HBaseTestingUtil.java:576)
"IPC Parameter Sending Thread for localhost/127.0.0.1:34433" daemon prio=5 tid=110 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
        at java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server idle connection scanner for port 38083" daemon prio=5 tid=96 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
        at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
"AsyncFSWAL-0-hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2-prefix:aa9080fe83d2,46375,1733472065852.meta" daemon prio=5 tid=298 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x5f18e5dc" daemon prio=5 tid=516 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Time-limited test-EventThread" daemon prio=5 tid=190 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
"ReadOnlyZKClient-127.0.0.1:64333@0x0d51575e" daemon prio=5 tid=492 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor@3b80ed4d" daemon prio=5 tid=61 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor.run(PendingReconstructionBlocks.java:267)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Registry-endpoints-refresh-end-points" daemon prio=5 tid=302 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hbase.client.RegistryEndpointsRefresher.mainLoop(RegistryEndpointsRefresher.java:92)
        at app//org.apache.hadoop.hbase.client.RegistryEndpointsRefresher$$Lambda$806/0x00007f6f68bb1750.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-10" daemon prio=5 tid=166 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server handler 3 on default port 34433" daemon prio=5 tid=67 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"IPC Server handler 4 on default port 38083" daemon prio=5 tid=108 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"RPCClient-NioEventLoopGroup-4-3" daemon prio=5 tid=306 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b52a47f2-08b3-21f1-9782-517d8d4597fe/cluster_acdf4e60-e0ee-bbdc-8ae2-647dd3ea8ddc/data/data2/current/BP-367880231-172.17.0.2-1733472062403" daemon prio=5 tid=124 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RPCClient-NioEventLoopGroup-4-5" daemon prio=5 tid=309 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server handler 4 on default port 34433" daemon prio=5 tid=68 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"Finalizer" daemon prio=8 tid=3 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
        at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
        at java.base@17.0.11/java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:172)
"PacketResponder: BP-367880231-172.17.0.2-1733472062403:blk_1073741833_1009, type=LAST_IN_PIPELINE" daemon prio=5 tid=287 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.Object.wait(Object.java:338)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Monitor thread for TaskMonitor" daemon prio=5 tid=210 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RS_OPEN_META-regionserver/aa9080fe83d2:0-0" daemon prio=5 tid=295 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x15b31be5" daemon prio=5 tid=697 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"pool-11-thread-1"  prio=5 tid=69 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"MiniHBaseClusterRegionServer-EventLoopGroup-3-1" daemon prio=10 tid=187 runnable
java.lang.Thread.State: RUNNABLE
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46375" daemon prio=5 tid=201 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"master/aa9080fe83d2:0:becomeActiveMaster-HFileCleaner.small.0-1733472068076" daemon prio=5 tid=269 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535)
        at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner.consumerLoop(HFileCleaner.java:285)
        at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner$2.run(HFileCleaner.java:269)
"RPCClient-NioEventLoopGroup-4-2" daemon prio=5 tid=305 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Session-HouseKeeper-49f404ad-1"  prio=5 tid=91 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server handler 3 on default port 38083" daemon prio=5 tid=107 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"ReadOnlyZKClient-127.0.0.1:64333@0x564f483c" daemon prio=5 tid=600 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@65e407ea" daemon prio=5 tid=85 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method)
        at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
        at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
        at java.base@17.0.11/sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:126)
        at app//org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:242)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"PEWorker-2" daemon prio=5 tid=248 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149)
        at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176)
"ReadOnlyZKClient-127.0.0.1:64333@0x2a17d2ee" daemon prio=5 tid=687 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"CacheReplicationMonitor(255963247)" daemon prio=5 tid=75 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:186)
"IPC Server handler 0 on default port 38083" daemon prio=5 tid=104 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"ReadOnlyZKClient-127.0.0.1:64333@0x59a9fd5a" daemon prio=5 tid=423 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x607c121a" daemon prio=5 tid=640 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"PEWorker-4" daemon prio=5 tid=250 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149)
        at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176)
"MiniHBaseClusterRegionServer-EventLoopGroup-3-2" daemon prio=10 tid=235 runnable
java.lang.Thread.State: RUNNABLE
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"DataXceiver for client DFSClient_NONMAPREDUCE_820960735_22 at /127.0.0.1:54212 [Receiving block BP-367880231-172.17.0.2-1733472062403:blk_1073741834_1010]" daemon prio=5 tid=296 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
        at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
        at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
        at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
        at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
        at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151)
        at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"SyncThread:0" daemon prio=5 tid=142 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
"weak-ref-cleaner-strictcontextstorage" daemon prio=1 tid=152 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
        at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
        at app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"HMaster-EventLoopGroup-1-2" daemon prio=10 tid=260 runnable
java.lang.Thread.State: RUNNABLE
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
        at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ProcExecTimeout" daemon prio=5 tid=245 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:279)
        at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81)
        at app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56)
"PacketResponder: BP-367880231-172.17.0.2-1733472062403:blk_1073741843_1019, type=LAST_IN_PIPELINE" daemon prio=5 tid=889 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.Object.wait(Object.java:338)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-12" daemon prio=5 tid=168 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x397280e5" daemon prio=5 tid=463 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOServerCxnFactory.AcceptThread:localhost/127.0.0.1:64333" daemon prio=5 tid=140 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
"Time-limited test" daemon prio=5 tid=22 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1864)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1898)
        at java.base@17.0.11/java.util.concurrent.CompletableFuture.join(CompletableFuture.java:2117)
        at app//org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi.clearPeerAndQueues(TestAsyncReplicationAdminApi.java:97)
        at java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
        at java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568)
        at app//org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
        at app//org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
        at app//org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
        at app//org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
        at app//org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
        at app//org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
        at app//org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
        at app//org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
        at app//org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
        at app//org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
        at app//org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
        at app//org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
        at app//org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
        at app//org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
        at app//org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
        at app//org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
        at app//org.junit.runners.ParentRunner.run(ParentRunner.java:413)
        at app//org.junit.runners.Suite.runChild(Suite.java:128)
        at app//org.junit.runners.Suite.runChild(Suite.java:27)
        at app//org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
        at app//org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
        at app//org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
        at app//org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
        at app//org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
        at app//org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
        at app//org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
        at app//org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
        at app//org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
        at java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"DataXceiver for client DFSClient_NONMAPREDUCE_820960735_22 at /127.0.0.1:54202 [Receiving block BP-367880231-172.17.0.2-1733472062403:blk_1073741833_1009]" daemon prio=5 tid=286 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
        at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
        at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
        at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
        at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
        at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
        at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151)
        at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176)
        at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110)
        at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x6082f3ae" daemon prio=5 tid=553 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x4e89f239" daemon prio=5 tid=522 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=39057" daemon prio=5 tid=182 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"Time-limited test.LruBlockCache.EvictionThread" daemon prio=5 tid=192 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:957)
"ReadOnlyZKClient-127.0.0.1:64333@0x4b5dee4f" daemon prio=5 tid=620 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-9" daemon prio=5 tid=165 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x0bd2d84c" daemon prio=5 tid=365 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"AsyncFSWAL-0-hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2-prefix:aa9080fe83d2,46375,1733472065852.rep" daemon prio=5 tid=328 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RS_OPEN_PRIORITY_REGION-regionserver/aa9080fe83d2:0-0" daemon prio=5 tid=325 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RS_COMPACTED_FILES_DISCHARGER-regionserver/aa9080fe83d2:0-0" daemon prio=5 tid=748 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-16" daemon prio=5 tid=172 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x079650c1" daemon prio=5 tid=590 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Time-limited test-SendThread(127.0.0.1:64333)" daemon prio=5 tid=155 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
"ReadOnlyZKClient-127.0.0.1:64333@0x2f80f580" daemon prio=5 tid=537 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x62df8b56" daemon prio=5 tid=630 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcClient-timer-pool-0" daemon prio=5 tid=243 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
        at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"master/aa9080fe83d2:0:becomeActiveMaster-HFileCleaner.large.0-1733472068075" daemon prio=5 tid=268 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at app//org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:101)
        at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner.consumerLoop(HFileCleaner.java:285)
        at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner$1.run(HFileCleaner.java:254)
"ReadOnlyZKClient-127.0.0.1:64333@0x572f5e25" daemon prio=5 tid=533 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"master/aa9080fe83d2:0.Chore.1" daemon prio=5 tid=262 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"JvmPauseMonitor" daemon prio=5 tid=277 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hbase.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:148)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"master:store-WAL-Roller" daemon prio=5 tid=226 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180)
"org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@6e3fca1c" daemon prio=5 tid=49 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:563)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x5a51960b" daemon prio=5 tid=603 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=5 tid=23 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
        at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
        at app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp1269359400-44" daemon prio=5 tid=44 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"WorkerMonitor" daemon prio=5 tid=246 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:279)
        at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81)
        at app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56)
"ReadOnlyZKClient-127.0.0.1:64333@0x5408e30e" daemon prio=5 tid=637 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"regionserver/aa9080fe83d2:0.Chore.1" daemon prio=5 tid=279 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"regionserver/aa9080fe83d2:0.leaseChecker" daemon prio=5 tid=280 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)
"RPCClient-NioEventLoopGroup-4-1" daemon prio=5 tid=303 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
        at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x24902694" daemon prio=5 tid=577 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server Responder" daemon prio=5 tid=57 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733)
        at app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716)
"qtp1269359400-43" daemon prio=5 tid=43 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"regionserver/aa9080fe83d2:0.procedureResultReporter" daemon prio=5 tid=282 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)
"region-location-0" daemon prio=5 tid=301 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-11" daemon prio=5 tid=167 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x401ee957" daemon prio=5 tid=335 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.hdfs.PeerCache@60a5d36c" daemon prio=5 tid=216 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
        at app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
        at app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46375" daemon prio=5 tid=197 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"IPC Server handler 2 on default port 34433" daemon prio=5 tid=66 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
        at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
        at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
"ReadOnlyZKClient-127.0.0.1:64333@0x63650f7f" daemon prio=5 tid=398 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"pool-1-thread-1" daemon prio=5 tid=14 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
        at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.run(EPollPort.java:275)
        at java.base@17.0.11/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:113)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOServerCxnFactory.SelectorThread-0" daemon prio=5 tid=138 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
"ReadOnlyZKClient-127.0.0.1:64333@0x1cc997fb" daemon prio=5 tid=417 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x2a8fcdf0" daemon prio=5 tid=731 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x4f302e4f" daemon prio=5 tid=573 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x19255325" daemon prio=5 tid=675 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x6cb68f69" daemon prio=5 tid=563 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"main"  prio=5 tid=1 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/java.lang.Thread.dumpThreads(Native Method)
        at java.base@17.0.11/java.lang.Thread.getAllStackTraces(Thread.java:1671)
        at app//org.apache.hadoop.hbase.TimedOutTestsListener.buildThreadDump(TimedOutTestsListener.java:92)
        at app//org.apache.hadoop.hbase.TimedOutTestsListener.buildThreadDiagnosticString(TimedOutTestsListener.java:78)
        at app//org.apache.hadoop.hbase.TimedOutTestsListener.testFailure(TimedOutTestsListener.java:65)
        at app//org.junit.runner.notification.SynchronizedRunListener.testFailure(SynchronizedRunListener.java:94)
        at app//org.junit.runner.notification.RunNotifier$6.notifyListener(RunNotifier.java:177)
        at app//org.junit.runner.notification.RunNotifier$SafeNotifier.run(RunNotifier.java:72)
        at app//org.junit.runner.notification.RunNotifier.fireTestFailures(RunNotifier.java:173)
        at app//org.junit.runner.notification.RunNotifier.fireTestFailure(RunNotifier.java:167)
        at app//org.apache.maven.surefire.common.junit4.Notifier.fireTestFailure(Notifier.java:100)
        at app//org.junit.internal.runners.model.EachTestNotifier.addFailure(EachTestNotifier.java:23)
        at app//org.junit.internal.runners.model.EachTestNotifier.addMultipleFailureException(EachTestNotifier.java:29)
        at app//org.junit.internal.runners.model.EachTestNotifier.addFailure(EachTestNotifier.java:21)
        at app//org.junit.runners.ParentRunner.run(ParentRunner.java:419)
        at app//org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:316)
        at app//org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:240)
        at app//org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:214)
        at app//org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:155)
        at app//org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:385)
        at app//org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162)
        at app//org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:507)
        at app//org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:495)
"LeaseRenewer:jenkins.hfs.0@localhost:34433" daemon prio=5 tid=288 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
        at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
        at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ProcessThread(sid:0 cport:64333):" daemon prio=5 tid=143 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
"MemStoreFlusher.0" daemon prio=5 tid=283 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:77)
        at app//org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:323)
"PacketResponder: BP-367880231-172.17.0.2-1733472062403:blk_1073741837_1013, type=LAST_IN_PIPELINE" daemon prio=5 tid=327 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.Object.wait(Object.java:338)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367)
        at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"IPC Server idle connection scanner for port 34433" daemon prio=5 tid=56 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
        at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
"ReadOnlyZKClient-127.0.0.1:64333@0x1f0ef537" daemon prio=5 tid=502 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"BootstrapNodeManager" daemon prio=5 tid=258 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=39057" daemon prio=5 tid=181 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39057" daemon prio=5 tid=179 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@147251d3" daemon prio=5 tid=72 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:4550)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"M:0;aa9080fe83d2:39057" daemon prio=5 tid=184 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:81)
        at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:64)
        at app//org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:625)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOWorkerThread-3" daemon prio=5 tid=158 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"pool-5-thread-1"  prio=5 tid=36 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x217b633e" daemon prio=5 tid=721 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=39057" daemon prio=5 tid=183 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"Hadoop-Metrics-Updater-0" daemon prio=5 tid=98 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x2d90ec8a" daemon prio=5 tid=617 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x07ac0126" daemon prio=5 tid=647 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x0a0dc73d" daemon prio=5 tid=711 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46375" daemon prio=5 tid=200 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39057" daemon prio=5 tid=176 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
        at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
        at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
"ReadOnlyZKClient-127.0.0.1:64333@0x31ffd422" daemon prio=5 tid=440 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x62c6a58c" daemon prio=5 tid=446 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"pool-14-thread-1" daemon prio=5 tid=132 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"pool-19-thread-1"  prio=5 tid=103 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Time-limited test.named-queue-events-pool-0" daemon prio=5 tid=186 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
        at app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
        at app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
        at app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"NIOServerCxnFactory.SelectorThread-1" daemon prio=5 tid=139 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
"snapshot-hfile-cleaner-cache-refresher" daemon prio=5 tid=265 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
        at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
"NIOWorkerThread-1" daemon prio=5 tid=145 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp1269359400-41-acceptor-0@76e83e1c-ServerConnector@135deabe{HTTP/1.1, (http/1.1)}{localhost:34833}" daemon prio=3 tid=41 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method)
        at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
        at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x56b58936" daemon prio=5 tid=361 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"surefire-forkedjvm-stream-flusher" daemon prio=5 tid=16 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"AsyncFSWAL-0-hdfs://localhost:34433/user/jenkins/test-data/9f953d1a-5916-441d-84bd-fe41fba7d3c2/MasterData-prefix:aa9080fe83d2,39057,1733472065171" daemon prio=5 tid=238 in Object.wait()
java.lang.Thread.State: WAITING (on object monitor)
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
        at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"qtp878742499-90" daemon prio=5 tid=90 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"RedundancyMonitor" daemon prio=5 tid=47 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at java.base@17.0.11/java.lang.Thread.sleep(Thread.java:344)
        at java.base@17.0.11/java.util.concurrent.TimeUnit.sleep(TimeUnit.java:446)
        at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$RedundancyMonitor.run(BlockManager.java:5352)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x03800144" daemon prio=5 tid=375 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"Common-Cleaner" daemon prio=8 tid=12 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Object.wait(Native Method)
        at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
        at java.base@17.0.11/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:140)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
        at java.base@17.0.11/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:162)
"org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@5db2f005" daemon prio=5 tid=74 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:4689)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"PEWorker-5" daemon prio=5 tid=251 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167)
        at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149)
        at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176)
"qtp1269359400-40" daemon prio=5 tid=40 runnable
java.lang.Thread.State: RUNNABLE
        at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f6f6842a550.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"ReadOnlyZKClient-127.0.0.1:64333@0x290cb819" daemon prio=5 tid=436 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
        at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
        at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$860/0x00007f6f68be74d0.run(Unknown Source)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
"org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@7bc5ad32" daemon prio=5 tid=73 timed_waiting
java.lang.Thread.State: TIMED_WAITING
        at java.base@17.0.11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:4592)
        at java.base@17.0.11/java.lang.Thread.run(Thread.java:840)