2024-12-11 23:19:26,672 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec 2024-12-11 23:19:26,683 main DEBUG Took 0.009909 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-11 23:19:26,684 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-11 23:19:26,684 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-11 23:19:26,685 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-11 23:19:26,686 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 23:19:26,700 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-11 23:19:26,712 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,714 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 23:19:26,715 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,715 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 23:19:26,716 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,716 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 23:19:26,717 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,718 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 23:19:26,718 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,719 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 23:19:26,720 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 23:19:26,720 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,721 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 23:19:26,721 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,722 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 23:19:26,722 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,723 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 23:19:26,723 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,724 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 23:19:26,724 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,724 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 23:19:26,725 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,725 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 23:19:26,726 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,726 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-11 23:19:26,728 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 23:19:26,730 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-11 23:19:26,733 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-11 23:19:26,733 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-11 23:19:26,735 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-11 23:19:26,735 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-11 23:19:26,746 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-11 23:19:26,749 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-11 23:19:26,752 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-11 23:19:26,753 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-11 23:19:26,753 main DEBUG createAppenders(={Console}) 2024-12-11 23:19:26,754 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec initialized 2024-12-11 23:19:26,755 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec 2024-12-11 23:19:26,755 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec OK. 2024-12-11 23:19:26,756 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-11 23:19:26,757 main DEBUG OutputStream closed 2024-12-11 23:19:26,757 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-11 23:19:26,757 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-11 23:19:26,758 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4efc180e OK 2024-12-11 23:19:26,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-11 23:19:26,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-11 23:19:26,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-11 23:19:26,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-11 23:19:26,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-11 23:19:26,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-11 23:19:26,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-11 23:19:26,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-11 23:19:26,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-11 23:19:26,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-11 23:19:26,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-11 23:19:26,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-11 23:19:26,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-11 23:19:26,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-11 23:19:26,848 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-11 23:19:26,848 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-11 23:19:26,848 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-11 23:19:26,849 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-11 23:19:26,851 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11 23:19:26,851 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@39de3d36) with optional ClassLoader: null 2024-12-11 23:19:26,851 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-11 23:19:26,852 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@39de3d36] started OK. 2024-12-11T23:19:27,098 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141 2024-12-11 23:19:27,100 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-11 23:19:27,101 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11T23:19:27,124 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi timeout: 26 mins 2024-12-11T23:19:27,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-11T23:19:27,153 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/cluster_e4efebb6-834a-ad19-6c5c-52938933ef0f, deleteOnExit=true 2024-12-11T23:19:27,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-11T23:19:27,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/test.cache.data in system properties and HBase conf 2024-12-11T23:19:27,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/hadoop.tmp.dir in system properties and HBase conf 2024-12-11T23:19:27,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/hadoop.log.dir in system properties and HBase conf 2024-12-11T23:19:27,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-11T23:19:27,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-11T23:19:27,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-11T23:19:27,235 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-11T23:19:27,325 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T23:19:27,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-11T23:19:27,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-11T23:19:27,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-11T23:19:27,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T23:19:27,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-11T23:19:27,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-11T23:19:27,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T23:19:27,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T23:19:27,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-11T23:19:27,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/nfs.dump.dir in system properties and HBase conf 2024-12-11T23:19:27,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/java.io.tmpdir in system properties and HBase conf 2024-12-11T23:19:27,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T23:19:27,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-11T23:19:27,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-11T23:19:28,252 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-11T23:19:28,326 INFO [Time-limited test {}] log.Log(170): Logging initialized @2265ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-11T23:19:28,390 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T23:19:28,447 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T23:19:28,465 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T23:19:28,465 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T23:19:28,467 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-11T23:19:28,478 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T23:19:28,481 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51976bf9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/hadoop.log.dir/,AVAILABLE} 2024-12-11T23:19:28,482 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@584a5323{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T23:19:28,639 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@235e335d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/java.io.tmpdir/jetty-localhost-39787-hadoop-hdfs-3_4_1-tests_jar-_-any-7982749371387185008/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-11T23:19:28,645 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1af3d2d7{HTTP/1.1, (http/1.1)}{localhost:39787} 2024-12-11T23:19:28,646 INFO [Time-limited test {}] server.Server(415): Started @2586ms 2024-12-11T23:19:29,148 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T23:19:29,154 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T23:19:29,157 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T23:19:29,157 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T23:19:29,157 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-11T23:19:29,158 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@60934148{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/hadoop.log.dir/,AVAILABLE} 2024-12-11T23:19:29,159 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c06ecae{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T23:19:29,257 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@ded0a8e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/java.io.tmpdir/jetty-localhost-35351-hadoop-hdfs-3_4_1-tests_jar-_-any-16423480860078577073/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T23:19:29,258 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@41fba10{HTTP/1.1, (http/1.1)}{localhost:35351} 2024-12-11T23:19:29,258 INFO [Time-limited test {}] server.Server(415): Started @3198ms 2024-12-11T23:19:29,305 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-11T23:19:30,261 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/cluster_e4efebb6-834a-ad19-6c5c-52938933ef0f/data/data2/current/BP-1905828385-172.17.0.2-1733959167787/current, will proceed with Du for space computation calculation, 2024-12-11T23:19:30,261 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/cluster_e4efebb6-834a-ad19-6c5c-52938933ef0f/data/data1/current/BP-1905828385-172.17.0.2-1733959167787/current, will proceed with Du for space computation calculation, 2024-12-11T23:19:30,291 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-11T23:19:30,329 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xea3ca75ada1c9d62 with lease ID 0xe3b54af1608fa8cf: Processing first storage report for DS-ec823315-b9ca-45af-9c64-68e2274b0859 from datanode DatanodeRegistration(127.0.0.1:44285, datanodeUuid=448f92f8-ea4a-4df5-b143-64350322717b, infoPort=43107, infoSecurePort=0, ipcPort=35819, storageInfo=lv=-57;cid=testClusterID;nsid=389278828;c=1733959167787) 2024-12-11T23:19:30,330 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xea3ca75ada1c9d62 with lease ID 0xe3b54af1608fa8cf: from storage DS-ec823315-b9ca-45af-9c64-68e2274b0859 node DatanodeRegistration(127.0.0.1:44285, datanodeUuid=448f92f8-ea4a-4df5-b143-64350322717b, infoPort=43107, infoSecurePort=0, ipcPort=35819, storageInfo=lv=-57;cid=testClusterID;nsid=389278828;c=1733959167787), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-11T23:19:30,330 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xea3ca75ada1c9d62 with lease ID 0xe3b54af1608fa8cf: Processing first storage report for DS-3326caa1-1811-4495-8dfc-cbc5eb3ad324 from datanode DatanodeRegistration(127.0.0.1:44285, datanodeUuid=448f92f8-ea4a-4df5-b143-64350322717b, infoPort=43107, infoSecurePort=0, ipcPort=35819, storageInfo=lv=-57;cid=testClusterID;nsid=389278828;c=1733959167787) 2024-12-11T23:19:30,330 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xea3ca75ada1c9d62 with lease ID 0xe3b54af1608fa8cf: from storage DS-3326caa1-1811-4495-8dfc-cbc5eb3ad324 node DatanodeRegistration(127.0.0.1:44285, datanodeUuid=448f92f8-ea4a-4df5-b143-64350322717b, infoPort=43107, infoSecurePort=0, ipcPort=35819, storageInfo=lv=-57;cid=testClusterID;nsid=389278828;c=1733959167787), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-11T23:19:30,391 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141 2024-12-11T23:19:30,452 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/cluster_e4efebb6-834a-ad19-6c5c-52938933ef0f/zookeeper_0, clientPort=50115, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/cluster_e4efebb6-834a-ad19-6c5c-52938933ef0f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/cluster_e4efebb6-834a-ad19-6c5c-52938933ef0f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-11T23:19:30,461 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50115 2024-12-11T23:19:30,471 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T23:19:30,474 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T23:19:30,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741825_1001 (size=7) 2024-12-11T23:19:31,080 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187 with version=8 2024-12-11T23:19:31,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/hbase-staging 2024-12-11T23:19:31,159 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-11T23:19:31,397 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d462fa545078:0 server-side Connection retries=6 2024-12-11T23:19:31,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T23:19:31,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T23:19:31,409 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T23:19:31,409 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T23:19:31,410 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T23:19:31,538 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-11T23:19:31,597 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-11T23:19:31,606 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-11T23:19:31,610 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T23:19:31,636 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 104166 (auto-detected) 2024-12-11T23:19:31,637 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-11T23:19:31,655 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41885 2024-12-11T23:19:31,674 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41885 connecting to ZooKeeper ensemble=127.0.0.1:50115 2024-12-11T23:19:31,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:418850x0, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T23:19:31,790 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41885-0x100172d404c0000 connected 2024-12-11T23:19:31,904 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T23:19:31,906 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T23:19:31,915 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T23:19:31,918 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187, hbase.cluster.distributed=false 2024-12-11T23:19:31,938 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T23:19:31,943 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41885 2024-12-11T23:19:31,943 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41885 2024-12-11T23:19:31,944 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41885 2024-12-11T23:19:31,944 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41885 2024-12-11T23:19:31,945 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41885 2024-12-11T23:19:32,040 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d462fa545078:0 server-side Connection retries=6 2024-12-11T23:19:32,041 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T23:19:32,042 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T23:19:32,042 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T23:19:32,042 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T23:19:32,042 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T23:19:32,045 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T23:19:32,048 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T23:19:32,048 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39401 2024-12-11T23:19:32,050 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39401 connecting to ZooKeeper ensemble=127.0.0.1:50115 2024-12-11T23:19:32,051 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T23:19:32,054 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T23:19:32,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:394010x0, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T23:19:32,072 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:394010x0, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T23:19:32,072 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39401-0x100172d404c0001 connected 2024-12-11T23:19:32,077 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T23:19:32,085 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T23:19:32,087 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T23:19:32,092 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T23:19:32,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39401 2024-12-11T23:19:32,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39401 2024-12-11T23:19:32,094 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39401 2024-12-11T23:19:32,094 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39401 2024-12-11T23:19:32,095 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39401 2024-12-11T23:19:32,110 DEBUG [M:0;d462fa545078:41885 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d462fa545078:41885 2024-12-11T23:19:32,111 INFO [master/d462fa545078:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d462fa545078,41885,1733959171253 2024-12-11T23:19:32,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T23:19:32,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T23:19:32,127 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d462fa545078,41885,1733959171253 2024-12-11T23:19:32,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:32,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T23:19:32,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:32,156 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-11T23:19:32,157 INFO [master/d462fa545078:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d462fa545078,41885,1733959171253 from backup master directory 2024-12-11T23:19:32,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d462fa545078,41885,1733959171253 2024-12-11T23:19:32,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T23:19:32,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T23:19:32,172 WARN [master/d462fa545078:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T23:19:32,172 INFO [master/d462fa545078:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d462fa545078,41885,1733959171253 2024-12-11T23:19:32,175 INFO [master/d462fa545078:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-11T23:19:32,177 INFO [master/d462fa545078:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-11T23:19:32,227 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/hbase.id] with ID: 7c7d4594-c434-4a58-99e5-d3b675ab8ed8 2024-12-11T23:19:32,227 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/.tmp/hbase.id 2024-12-11T23:19:32,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741826_1002 (size=42) 2024-12-11T23:19:32,643 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/.tmp/hbase.id]:[hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/hbase.id] 2024-12-11T23:19:32,695 INFO [master/d462fa545078:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T23:19:32,699 INFO [master/d462fa545078:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-11T23:19:32,716 INFO [master/d462fa545078:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-11T23:19:32,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:32,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:32,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741827_1003 (size=196) 2024-12-11T23:19:33,202 INFO [master/d462fa545078:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T23:19:33,203 INFO [master/d462fa545078:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-11T23:19:33,216 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:33,219 INFO [master/d462fa545078:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T23:19:33,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741828_1004 (size=1189) 2024-12-11T23:19:33,670 INFO [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store 2024-12-11T23:19:33,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741829_1005 (size=34) 2024-12-11T23:19:34,099 INFO [master/d462fa545078:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-11T23:19:34,105 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T23:19:34,106 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T23:19:34,106 INFO [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T23:19:34,106 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T23:19:34,108 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T23:19:34,108 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T23:19:34,108 INFO [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T23:19:34,109 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733959174106Disabling compacts and flushes for region at 1733959174106Disabling writes for close at 1733959174108 (+2 ms)Writing region close event to WAL at 1733959174108Closed at 1733959174108 2024-12-11T23:19:34,111 WARN [master/d462fa545078:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/.initializing 2024-12-11T23:19:34,111 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/WALs/d462fa545078,41885,1733959171253 2024-12-11T23:19:34,118 INFO [master/d462fa545078:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T23:19:34,131 INFO [master/d462fa545078:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d462fa545078%2C41885%2C1733959171253, suffix=, logDir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/WALs/d462fa545078,41885,1733959171253, archiveDir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/oldWALs, maxLogs=10 2024-12-11T23:19:34,149 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/WALs/d462fa545078,41885,1733959171253/d462fa545078%2C41885%2C1733959171253.1733959174134, exclude list is [], retry=0 2024-12-11T23:19:34,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44285,DS-ec823315-b9ca-45af-9c64-68e2274b0859,DISK] 2024-12-11T23:19:34,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-11T23:19:34,196 INFO [master/d462fa545078:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/WALs/d462fa545078,41885,1733959171253/d462fa545078%2C41885%2C1733959171253.1733959174134 2024-12-11T23:19:34,197 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43107:43107)] 2024-12-11T23:19:34,198 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-11T23:19:34,198 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T23:19:34,201 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T23:19:34,202 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T23:19:34,235 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T23:19:34,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-11T23:19:34,262 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:34,264 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T23:19:34,265 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T23:19:34,268 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-11T23:19:34,268 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:34,269 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T23:19:34,270 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T23:19:34,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-11T23:19:34,272 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:34,273 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T23:19:34,274 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T23:19:34,276 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-11T23:19:34,277 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:34,278 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T23:19:34,278 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T23:19:34,281 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T23:19:34,282 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T23:19:34,287 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T23:19:34,288 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T23:19:34,291 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T23:19:34,295 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T23:19:34,300 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T23:19:34,302 INFO [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72507047, jitterRate=0.08043919503688812}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T23:19:34,309 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733959174213Initializing all the Stores at 1733959174215 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733959174215Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733959174216 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733959174216Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733959174216Cleaning up temporary data from old regions at 1733959174288 (+72 ms)Region opened successfully at 1733959174309 (+21 ms) 2024-12-11T23:19:34,311 INFO [master/d462fa545078:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-11T23:19:34,343 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c81e5b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d462fa545078/172.17.0.2:0 2024-12-11T23:19:34,374 INFO [master/d462fa545078:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-11T23:19:34,383 INFO [master/d462fa545078:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-11T23:19:34,383 INFO [master/d462fa545078:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-11T23:19:34,386 INFO [master/d462fa545078:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T23:19:34,387 INFO [master/d462fa545078:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-11T23:19:34,393 INFO [master/d462fa545078:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-11T23:19:34,394 INFO [master/d462fa545078:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-11T23:19:34,422 INFO [master/d462fa545078:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-11T23:19:34,432 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-11T23:19:34,534 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-11T23:19:34,539 INFO [master/d462fa545078:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-11T23:19:34,541 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-11T23:19:34,549 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-11T23:19:34,552 INFO [master/d462fa545078:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-11T23:19:34,556 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-11T23:19:34,565 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-11T23:19:34,568 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-11T23:19:34,581 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-11T23:19:34,608 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-11T23:19:34,618 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-11T23:19:34,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T23:19:34,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T23:19:34,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:34,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:34,637 INFO [master/d462fa545078:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d462fa545078,41885,1733959171253, sessionid=0x100172d404c0000, setting cluster-up flag (Was=false) 2024-12-11T23:19:34,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:34,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:34,697 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-11T23:19:34,699 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d462fa545078,41885,1733959171253 2024-12-11T23:19:34,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:34,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:34,755 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-11T23:19:34,760 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d462fa545078,41885,1733959171253 2024-12-11T23:19:34,770 INFO [master/d462fa545078:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-11T23:19:34,801 INFO [RS:0;d462fa545078:39401 {}] regionserver.HRegionServer(746): ClusterId : 7c7d4594-c434-4a58-99e5-d3b675ab8ed8 2024-12-11T23:19:34,804 DEBUG [RS:0;d462fa545078:39401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T23:19:34,815 DEBUG [RS:0;d462fa545078:39401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T23:19:34,815 DEBUG [RS:0;d462fa545078:39401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T23:19:34,825 DEBUG [RS:0;d462fa545078:39401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T23:19:34,825 DEBUG [RS:0;d462fa545078:39401 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e1e52b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d462fa545078/172.17.0.2:0 2024-12-11T23:19:34,837 DEBUG [RS:0;d462fa545078:39401 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d462fa545078:39401 2024-12-11T23:19:34,839 INFO [RS:0;d462fa545078:39401 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T23:19:34,840 INFO [RS:0;d462fa545078:39401 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T23:19:34,840 DEBUG [RS:0;d462fa545078:39401 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T23:19:34,842 INFO [RS:0;d462fa545078:39401 {}] regionserver.HRegionServer(2659): reportForDuty to master=d462fa545078,41885,1733959171253 with port=39401, startcode=1733959172007 2024-12-11T23:19:34,844 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T23:19:34,851 DEBUG [RS:0;d462fa545078:39401 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T23:19:34,853 INFO [master/d462fa545078:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-11T23:19:34,859 INFO [master/d462fa545078:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-11T23:19:34,865 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d462fa545078,41885,1733959171253 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-11T23:19:34,874 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d462fa545078:0, corePoolSize=5, maxPoolSize=5 2024-12-11T23:19:34,875 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d462fa545078:0, corePoolSize=5, maxPoolSize=5 2024-12-11T23:19:34,875 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d462fa545078:0, corePoolSize=5, maxPoolSize=5 2024-12-11T23:19:34,875 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d462fa545078:0, corePoolSize=5, maxPoolSize=5 2024-12-11T23:19:34,876 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d462fa545078:0, corePoolSize=10, maxPoolSize=10 2024-12-11T23:19:34,876 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d462fa545078:0, corePoolSize=1, maxPoolSize=1 2024-12-11T23:19:34,876 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d462fa545078:0, corePoolSize=2, maxPoolSize=2 2024-12-11T23:19:34,876 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d462fa545078:0, corePoolSize=1, maxPoolSize=1 2024-12-11T23:19:34,879 INFO [master/d462fa545078:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733959204879 2024-12-11T23:19:34,879 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56597, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T23:19:34,881 INFO [master/d462fa545078:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-11T23:19:34,882 INFO [master/d462fa545078:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-11T23:19:34,883 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T23:19:34,884 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-11T23:19:34,886 INFO [master/d462fa545078:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-11T23:19:34,886 INFO [master/d462fa545078:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-11T23:19:34,887 INFO [master/d462fa545078:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-11T23:19:34,887 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41885 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:34,887 INFO [master/d462fa545078:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-11T23:19:34,888 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:34,891 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:34,891 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-11T23:19:34,893 INFO [master/d462fa545078:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-11T23:19:34,894 INFO [master/d462fa545078:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-11T23:19:34,895 INFO [master/d462fa545078:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-11T23:19:34,897 INFO [master/d462fa545078:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-11T23:19:34,898 INFO [master/d462fa545078:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-11T23:19:34,900 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d462fa545078:0:becomeActiveMaster-HFileCleaner.large.0-1733959174899,5,FailOnTimeoutGroup] 2024-12-11T23:19:34,903 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d462fa545078:0:becomeActiveMaster-HFileCleaner.small.0-1733959174900,5,FailOnTimeoutGroup] 2024-12-11T23:19:34,904 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:34,904 INFO [master/d462fa545078:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-11T23:19:34,906 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:34,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741831_1007 (size=1321) 2024-12-11T23:19:34,906 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:34,917 DEBUG [RS:0;d462fa545078:39401 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-11T23:19:34,917 WARN [RS:0;d462fa545078:39401 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-11T23:19:35,019 INFO [RS:0;d462fa545078:39401 {}] regionserver.HRegionServer(2659): reportForDuty to master=d462fa545078,41885,1733959171253 with port=39401, startcode=1733959172007 2024-12-11T23:19:35,021 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41885 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d462fa545078,39401,1733959172007 2024-12-11T23:19:35,024 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41885 {}] master.ServerManager(517): Registering regionserver=d462fa545078,39401,1733959172007 2024-12-11T23:19:35,032 DEBUG [RS:0;d462fa545078:39401 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187 2024-12-11T23:19:35,033 DEBUG [RS:0;d462fa545078:39401 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36649 2024-12-11T23:19:35,033 DEBUG [RS:0;d462fa545078:39401 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T23:19:35,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T23:19:35,045 DEBUG [RS:0;d462fa545078:39401 {}] zookeeper.ZKUtil(111): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d462fa545078,39401,1733959172007 2024-12-11T23:19:35,045 WARN [RS:0;d462fa545078:39401 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T23:19:35,046 INFO [RS:0;d462fa545078:39401 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T23:19:35,046 DEBUG [RS:0;d462fa545078:39401 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/WALs/d462fa545078,39401,1733959172007 2024-12-11T23:19:35,048 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d462fa545078,39401,1733959172007] 2024-12-11T23:19:35,070 INFO [RS:0;d462fa545078:39401 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T23:19:35,081 INFO [RS:0;d462fa545078:39401 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T23:19:35,085 INFO [RS:0;d462fa545078:39401 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T23:19:35,085 INFO [RS:0;d462fa545078:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:35,086 INFO [RS:0;d462fa545078:39401 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T23:19:35,091 INFO [RS:0;d462fa545078:39401 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T23:19:35,092 INFO [RS:0;d462fa545078:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:35,092 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d462fa545078:0, corePoolSize=1, maxPoolSize=1 2024-12-11T23:19:35,093 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d462fa545078:0, corePoolSize=1, maxPoolSize=1 2024-12-11T23:19:35,093 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0, corePoolSize=1, maxPoolSize=1 2024-12-11T23:19:35,093 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d462fa545078:0, corePoolSize=1, maxPoolSize=1 2024-12-11T23:19:35,093 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d462fa545078:0, corePoolSize=1, maxPoolSize=1 2024-12-11T23:19:35,093 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d462fa545078:0, corePoolSize=2, maxPoolSize=2 2024-12-11T23:19:35,093 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d462fa545078:0, corePoolSize=1, maxPoolSize=1 2024-12-11T23:19:35,094 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d462fa545078:0, corePoolSize=1, maxPoolSize=1 2024-12-11T23:19:35,094 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d462fa545078:0, corePoolSize=1, maxPoolSize=1 2024-12-11T23:19:35,094 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d462fa545078:0, corePoolSize=1, maxPoolSize=1 2024-12-11T23:19:35,094 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d462fa545078:0, corePoolSize=1, maxPoolSize=1 2024-12-11T23:19:35,094 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d462fa545078:0, corePoolSize=1, maxPoolSize=1 2024-12-11T23:19:35,094 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d462fa545078:0, corePoolSize=3, maxPoolSize=3 2024-12-11T23:19:35,095 DEBUG [RS:0;d462fa545078:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d462fa545078:0, corePoolSize=3, maxPoolSize=3 2024-12-11T23:19:35,096 INFO [RS:0;d462fa545078:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:35,096 INFO [RS:0;d462fa545078:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:35,096 INFO [RS:0;d462fa545078:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:35,096 INFO [RS:0;d462fa545078:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:35,097 INFO [RS:0;d462fa545078:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:35,097 INFO [RS:0;d462fa545078:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=d462fa545078,39401,1733959172007-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T23:19:35,112 INFO [RS:0;d462fa545078:39401 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2024-12-11T23:19:35,114 INFO [RS:0;d462fa545078:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=d462fa545078,39401,1733959172007-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:35,114 INFO [RS:0;d462fa545078:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:35,114 INFO [RS:0;d462fa545078:39401 {}] regionserver.Replication(171): d462fa545078,39401,1733959172007 started 2024-12-11T23:19:35,129 INFO [RS:0;d462fa545078:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:35,129 INFO [RS:0;d462fa545078:39401 {}] regionserver.HRegionServer(1482): Serving as d462fa545078,39401,1733959172007, RpcServer on d462fa545078/172.17.0.2:39401, sessionid=0x100172d404c0001 2024-12-11T23:19:35,130 DEBUG [RS:0;d462fa545078:39401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T23:19:35,130 DEBUG [RS:0;d462fa545078:39401 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d462fa545078,39401,1733959172007 2024-12-11T23:19:35,130 DEBUG [RS:0;d462fa545078:39401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd462fa545078,39401,1733959172007' 2024-12-11T23:19:35,130 DEBUG [RS:0;d462fa545078:39401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T23:19:35,131 DEBUG [RS:0;d462fa545078:39401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T23:19:35,132 DEBUG [RS:0;d462fa545078:39401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T23:19:35,132 DEBUG [RS:0;d462fa545078:39401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T23:19:35,132 DEBUG [RS:0;d462fa545078:39401 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d462fa545078,39401,1733959172007 2024-12-11T23:19:35,132 DEBUG [RS:0;d462fa545078:39401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd462fa545078,39401,1733959172007' 2024-12-11T23:19:35,132 DEBUG [RS:0;d462fa545078:39401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T23:19:35,133 DEBUG [RS:0;d462fa545078:39401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T23:19:35,134 DEBUG [RS:0;d462fa545078:39401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T23:19:35,134 INFO [RS:0;d462fa545078:39401 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T23:19:35,134 INFO [RS:0;d462fa545078:39401 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T23:19:35,240 INFO [RS:0;d462fa545078:39401 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T23:19:35,245 INFO [RS:0;d462fa545078:39401 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d462fa545078%2C39401%2C1733959172007, suffix=, logDir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/WALs/d462fa545078,39401,1733959172007, archiveDir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/oldWALs, maxLogs=32 2024-12-11T23:19:35,260 DEBUG [RS:0;d462fa545078:39401 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/WALs/d462fa545078,39401,1733959172007/d462fa545078%2C39401%2C1733959172007.1733959175249, exclude list is [], retry=0 2024-12-11T23:19:35,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44285,DS-ec823315-b9ca-45af-9c64-68e2274b0859,DISK] 2024-12-11T23:19:35,267 INFO [RS:0;d462fa545078:39401 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/WALs/d462fa545078,39401,1733959172007/d462fa545078%2C39401%2C1733959172007.1733959175249 2024-12-11T23:19:35,267 DEBUG [RS:0;d462fa545078:39401 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43107:43107)] 2024-12-11T23:19:35,310 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-11T23:19:35,311 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187 2024-12-11T23:19:35,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741833_1009 (size=32) 2024-12-11T23:19:35,727 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T23:19:35,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T23:19:35,735 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T23:19:35,735 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:35,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T23:19:35,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T23:19:35,739 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T23:19:35,740 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:35,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T23:19:35,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T23:19:35,744 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T23:19:35,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:35,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T23:19:35,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T23:19:35,748 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T23:19:35,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:35,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T23:19:35,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T23:19:35,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740 2024-12-11T23:19:35,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740 2024-12-11T23:19:35,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T23:19:35,756 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T23:19:35,757 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T23:19:35,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T23:19:35,763 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T23:19:35,764 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61110023, jitterRate=-0.08938969671726227}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T23:19:35,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733959175727Initializing all the Stores at 1733959175731 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733959175731Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733959175732 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733959175732Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733959175732Cleaning up temporary data from old regions at 1733959175756 (+24 ms)Region opened successfully at 1733959175767 (+11 ms) 2024-12-11T23:19:35,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T23:19:35,767 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T23:19:35,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T23:19:35,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T23:19:35,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T23:19:35,769 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T23:19:35,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733959175767Disabling compacts and flushes for region at 1733959175767Disabling writes for close at 1733959175768 (+1 ms)Writing region close event to WAL at 1733959175769 (+1 ms)Closed at 1733959175769 2024-12-11T23:19:35,773 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T23:19:35,773 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-11T23:19:35,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-11T23:19:35,786 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T23:19:35,788 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-11T23:19:35,945 DEBUG [d462fa545078:41885 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-11T23:19:35,957 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d462fa545078,39401,1733959172007 2024-12-11T23:19:35,962 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d462fa545078,39401,1733959172007, state=OPENING 2024-12-11T23:19:36,039 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-11T23:19:36,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:36,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:36,051 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T23:19:36,051 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T23:19:36,052 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T23:19:36,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d462fa545078,39401,1733959172007}] 2024-12-11T23:19:36,236 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T23:19:36,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49767, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T23:19:36,249 INFO [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-11T23:19:36,250 INFO [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T23:19:36,250 INFO [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-11T23:19:36,254 INFO [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d462fa545078%2C39401%2C1733959172007.meta, suffix=.meta, logDir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/WALs/d462fa545078,39401,1733959172007, archiveDir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/oldWALs, maxLogs=32 2024-12-11T23:19:36,268 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/WALs/d462fa545078,39401,1733959172007/d462fa545078%2C39401%2C1733959172007.meta.1733959176256.meta, exclude list is [], retry=0 2024-12-11T23:19:36,272 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44285,DS-ec823315-b9ca-45af-9c64-68e2274b0859,DISK] 2024-12-11T23:19:36,274 INFO [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/WALs/d462fa545078,39401,1733959172007/d462fa545078%2C39401%2C1733959172007.meta.1733959176256.meta 2024-12-11T23:19:36,275 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43107:43107)] 2024-12-11T23:19:36,275 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-11T23:19:36,277 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-11T23:19:36,279 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-11T23:19:36,283 INFO [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-11T23:19:36,287 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-11T23:19:36,287 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T23:19:36,288 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-11T23:19:36,288 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-11T23:19:36,291 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T23:19:36,292 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T23:19:36,292 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:36,293 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T23:19:36,294 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T23:19:36,295 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T23:19:36,295 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:36,296 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T23:19:36,296 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T23:19:36,298 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T23:19:36,298 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:36,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T23:19:36,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T23:19:36,300 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T23:19:36,300 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:36,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T23:19:36,301 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T23:19:36,302 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740 2024-12-11T23:19:36,305 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740 2024-12-11T23:19:36,307 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T23:19:36,307 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T23:19:36,308 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T23:19:36,311 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T23:19:36,313 INFO [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74922272, jitterRate=0.11642885208129883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T23:19:36,313 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-11T23:19:36,314 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733959176288Writing region info on filesystem at 1733959176288Initializing all the Stores at 1733959176290 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733959176290Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733959176291 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733959176291Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733959176291Cleaning up temporary data from old regions at 1733959176307 (+16 ms)Running coprocessor post-open hooks at 1733959176313 (+6 ms)Region opened successfully at 1733959176314 (+1 ms) 2024-12-11T23:19:36,321 INFO [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733959176225 2024-12-11T23:19:36,334 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d462fa545078,39401,1733959172007 2024-12-11T23:19:36,336 DEBUG [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-11T23:19:36,336 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d462fa545078,39401,1733959172007, state=OPEN 2024-12-11T23:19:36,337 INFO [RS_OPEN_META-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-11T23:19:36,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T23:19:36,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T23:19:36,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T23:19:36,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T23:19:36,380 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d462fa545078,39401,1733959172007 2024-12-11T23:19:36,391 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-11T23:19:36,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d462fa545078,39401,1733959172007 in 326 msec 2024-12-11T23:19:36,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-11T23:19:36,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 614 msec 2024-12-11T23:19:36,401 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T23:19:36,401 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-11T23:19:36,419 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T23:19:36,420 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d462fa545078,39401,1733959172007, seqNum=-1] 2024-12-11T23:19:36,437 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T23:19:36,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51333, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T23:19:36,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6620 sec 2024-12-11T23:19:36,461 INFO [master/d462fa545078:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733959176461, completionTime=-1 2024-12-11T23:19:36,463 INFO [master/d462fa545078:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-11T23:19:36,463 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T23:19:36,483 INFO [master/d462fa545078:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-11T23:19:36,483 INFO [master/d462fa545078:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733959236483 2024-12-11T23:19:36,483 INFO [master/d462fa545078:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733959296483 2024-12-11T23:19:36,483 INFO [master/d462fa545078:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 19 msec 2024-12-11T23:19:36,485 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d462fa545078,41885,1733959171253-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:36,485 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d462fa545078,41885,1733959171253-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:36,485 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d462fa545078,41885,1733959171253-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:36,486 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d462fa545078:41885, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:36,487 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:36,487 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:36,493 DEBUG [master/d462fa545078:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T23:19:36,515 INFO [master/d462fa545078:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.343sec 2024-12-11T23:19:36,516 INFO [master/d462fa545078:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-11T23:19:36,517 INFO [master/d462fa545078:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-11T23:19:36,519 INFO [master/d462fa545078:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-11T23:19:36,519 INFO [master/d462fa545078:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-11T23:19:36,519 INFO [master/d462fa545078:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-11T23:19:36,520 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d462fa545078,41885,1733959171253-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T23:19:36,521 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d462fa545078,41885,1733959171253-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-11T23:19:36,529 DEBUG [master/d462fa545078:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-11T23:19:36,530 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-11T23:19:36,530 INFO [master/d462fa545078:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d462fa545078,41885,1733959171253-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T23:19:36,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23060167, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T23:19:36,634 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-11T23:19:36,634 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-11T23:19:36,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d462fa545078,41885,-1 for getting cluster id 2024-12-11T23:19:36,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T23:19:36,646 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7c7d4594-c434-4a58-99e5-d3b675ab8ed8' 2024-12-11T23:19:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T23:19:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7c7d4594-c434-4a58-99e5-d3b675ab8ed8" 2024-12-11T23:19:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffc8cfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T23:19:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d462fa545078,41885,-1] 2024-12-11T23:19:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T23:19:36,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T23:19:36,659 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T23:19:36,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8acdf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T23:19:36,662 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T23:19:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d462fa545078,39401,1733959172007, seqNum=-1] 2024-12-11T23:19:36,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T23:19:36,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T23:19:36,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d462fa545078,41885,1733959171253 2024-12-11T23:19:36,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f14dfc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T23:19:36,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d462fa545078,41885,-1 for getting cluster id 2024-12-11T23:19:36,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T23:19:36,694 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7c7d4594-c434-4a58-99e5-d3b675ab8ed8' 2024-12-11T23:19:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T23:19:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7c7d4594-c434-4a58-99e5-d3b675ab8ed8" 2024-12-11T23:19:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e98eb5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T23:19:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d462fa545078,41885,-1] 2024-12-11T23:19:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T23:19:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T23:19:36,697 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T23:19:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a83ddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T23:19:36,714 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=214, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=92, ProcessCount=11, AvailableMemoryMB=9104 2024-12-11T23:19:36,723 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T23:19:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is d462fa545078,41885,1733959171253 2024-12-11T23:19:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33f39997 2024-12-11T23:19:36,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T23:19:36,735 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T23:19:36,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:36,748 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T23:19:36,752 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3fc69874 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@52c8c7b2 2024-12-11T23:19:36,752 WARN [PEWorker-3 {}] client.ZKConnectionRegistry(87): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-12-11T23:19:36,799 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x3fc69874 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3fc69874 to 127.0.0.1:50115 2024-12-11T23:19:36,799 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:36,801 INFO [PEWorker-3 {}] master.HMaster(2490): Client=null/null create 'hbase:replication', {TABLE_ATTRIBUTES => {coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.regionserver.region.split_restriction.delimiter' => '-', 'hbase.regionserver.region.split_restriction.type' => 'DelimitedKeyPrefix'}}}, {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T23:19:36,805 DEBUG [PEWorker-3 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:replication 2024-12-11T23:19:36,809 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T23:19:36,810 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:36,812 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T23:19:36,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741835_1011 (size=1138) 2024-12-11T23:19:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T23:19:36,858 DEBUG [PEWorker-3 {}] procedure.ProcedureSyncWait(219): waitFor Creating table hbase:replication 2024-12-11T23:19:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T23:19:37,227 INFO [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 301a3ed59652d1a1ff0f40a1ccd90f05, NAME => 'hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:replication', {TABLE_ATTRIBUTES => {coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.regionserver.region.split_restriction.delimiter' => '-', 'hbase.regionserver.region.split_restriction.type' => 'DelimitedKeyPrefix', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187 2024-12-11T23:19:37,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741836_1012 (size=44) 2024-12-11T23:19:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T23:19:37,640 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T23:19:37,640 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1722): Closing 301a3ed59652d1a1ff0f40a1ccd90f05, disabling compactions & flushes 2024-12-11T23:19:37,640 INFO [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05. 2024-12-11T23:19:37,640 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05. 2024-12-11T23:19:37,640 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05. after waiting 0 ms 2024-12-11T23:19:37,640 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05. 2024-12-11T23:19:37,641 INFO [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1973): Closed hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05. 2024-12-11T23:19:37,641 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1676): Region close journal for 301a3ed59652d1a1ff0f40a1ccd90f05: Waiting for close lock at 1733959177640Disabling compacts and flushes for region at 1733959177640Disabling writes for close at 1733959177640Writing region close event to WAL at 1733959177640Closed at 1733959177640 2024-12-11T23:19:37,645 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T23:19:37,652 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05.","families":{"info":[{"qualifier":"regioninfo","vlen":43,"tag":[],"timestamp":"1733959177646"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733959177646"}]},"ts":"1733959177646"} 2024-12-11T23:19:37,657 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T23:19:37,659 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T23:19:37,661 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:replication","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733959177659"}]},"ts":"1733959177659"} 2024-12-11T23:19:37,665 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:replication, state=ENABLING in hbase:meta 2024-12-11T23:19:37,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=301a3ed59652d1a1ff0f40a1ccd90f05, ASSIGN}] 2024-12-11T23:19:37,669 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=301a3ed59652d1a1ff0f40a1ccd90f05, ASSIGN 2024-12-11T23:19:37,671 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:replication, region=301a3ed59652d1a1ff0f40a1ccd90f05, ASSIGN; state=OFFLINE, location=d462fa545078,39401,1733959172007; forceNewPlan=false, retain=false 2024-12-11T23:19:37,823 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=301a3ed59652d1a1ff0f40a1ccd90f05, regionState=OPENING, regionLocation=d462fa545078,39401,1733959172007 2024-12-11T23:19:37,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:replication, region=301a3ed59652d1a1ff0f40a1ccd90f05, ASSIGN because future has completed 2024-12-11T23:19:37,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=7, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 301a3ed59652d1a1ff0f40a1ccd90f05, server=d462fa545078,39401,1733959172007}] 2024-12-11T23:19:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T23:19:37,996 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] handler.AssignRegionHandler(132): Open hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05. 2024-12-11T23:19:37,996 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T23:19:37,997 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] monitor.StreamSlowMonitor(122): New stream slow monitor rep 2024-12-11T23:19:38,000 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d462fa545078%2C39401%2C1733959172007.rep, suffix=, logDir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/WALs/d462fa545078,39401,1733959172007, archiveDir=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/oldWALs, maxLogs=32 2024-12-11T23:19:38,014 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/WALs/d462fa545078,39401,1733959172007/d462fa545078%2C39401%2C1733959172007.rep.1733959178002, exclude list is [], retry=0 2024-12-11T23:19:38,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44285,DS-ec823315-b9ca-45af-9c64-68e2274b0859,DISK] 2024-12-11T23:19:38,020 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/WALs/d462fa545078,39401,1733959172007/d462fa545078%2C39401%2C1733959172007.rep.1733959178002 2024-12-11T23:19:38,020 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43107:43107)] 2024-12-11T23:19:38,021 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7752): Opening region: {ENCODED => 301a3ed59652d1a1ff0f40a1ccd90f05, NAME => 'hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05.', STARTKEY => '', ENDKEY => ''} 2024-12-11T23:19:38,021 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-11T23:19:38,021 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05. service=MultiRowMutationService 2024-12-11T23:19:38,021 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:replication successfully. 2024-12-11T23:19:38,021 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table replication 301a3ed59652d1a1ff0f40a1ccd90f05 2024-12-11T23:19:38,021 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(898): Instantiated hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T23:19:38,022 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7794): checking encryption for 301a3ed59652d1a1ff0f40a1ccd90f05 2024-12-11T23:19:38,022 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7797): checking classloading for 301a3ed59652d1a1ff0f40a1ccd90f05 2024-12-11T23:19:38,024 INFO [StoreOpener-301a3ed59652d1a1ff0f40a1ccd90f05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family hfileref of region 301a3ed59652d1a1ff0f40a1ccd90f05 2024-12-11T23:19:38,027 INFO [StoreOpener-301a3ed59652d1a1ff0f40a1ccd90f05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 301a3ed59652d1a1ff0f40a1ccd90f05 columnFamilyName hfileref 2024-12-11T23:19:38,027 DEBUG [StoreOpener-301a3ed59652d1a1ff0f40a1ccd90f05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:38,028 INFO [StoreOpener-301a3ed59652d1a1ff0f40a1ccd90f05-1 {}] regionserver.HStore(327): Store=301a3ed59652d1a1ff0f40a1ccd90f05/hfileref, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T23:19:38,028 INFO [StoreOpener-301a3ed59652d1a1ff0f40a1ccd90f05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family queue of region 301a3ed59652d1a1ff0f40a1ccd90f05 2024-12-11T23:19:38,031 INFO [StoreOpener-301a3ed59652d1a1ff0f40a1ccd90f05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 301a3ed59652d1a1ff0f40a1ccd90f05 columnFamilyName queue 2024-12-11T23:19:38,031 DEBUG [StoreOpener-301a3ed59652d1a1ff0f40a1ccd90f05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:38,032 INFO [StoreOpener-301a3ed59652d1a1ff0f40a1ccd90f05-1 {}] regionserver.HStore(327): Store=301a3ed59652d1a1ff0f40a1ccd90f05/queue, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T23:19:38,032 INFO [StoreOpener-301a3ed59652d1a1ff0f40a1ccd90f05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family sid of region 301a3ed59652d1a1ff0f40a1ccd90f05 2024-12-11T23:19:38,034 INFO [StoreOpener-301a3ed59652d1a1ff0f40a1ccd90f05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 301a3ed59652d1a1ff0f40a1ccd90f05 columnFamilyName sid 2024-12-11T23:19:38,034 DEBUG [StoreOpener-301a3ed59652d1a1ff0f40a1ccd90f05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T23:19:38,035 INFO [StoreOpener-301a3ed59652d1a1ff0f40a1ccd90f05-1 {}] regionserver.HStore(327): Store=301a3ed59652d1a1ff0f40a1ccd90f05/sid, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T23:19:38,035 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1038): replaying wal for 301a3ed59652d1a1ff0f40a1ccd90f05 2024-12-11T23:19:38,037 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/replication/301a3ed59652d1a1ff0f40a1ccd90f05 2024-12-11T23:19:38,037 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/replication/301a3ed59652d1a1ff0f40a1ccd90f05 2024-12-11T23:19:38,039 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1048): stopping wal replay for 301a3ed59652d1a1ff0f40a1ccd90f05 2024-12-11T23:19:38,039 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1060): Cleaning up temporary data for 301a3ed59652d1a1ff0f40a1ccd90f05 2024-12-11T23:19:38,040 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:replication descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T23:19:38,043 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1093): writing seq id for 301a3ed59652d1a1ff0f40a1ccd90f05 2024-12-11T23:19:38,047 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/replication/301a3ed59652d1a1ff0f40a1ccd90f05/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T23:19:38,048 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1114): Opened 301a3ed59652d1a1ff0f40a1ccd90f05; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67365152, jitterRate=0.003818988800048828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T23:19:38,048 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 301a3ed59652d1a1ff0f40a1ccd90f05 2024-12-11T23:19:38,049 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1006): Region open journal for 301a3ed59652d1a1ff0f40a1ccd90f05: Running coprocessor pre-open hook at 1733959178022Writing region info on filesystem at 1733959178022Initializing all the Stores at 1733959178024 (+2 ms)Instantiating store for column family {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733959178024Instantiating store for column family {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733959178024Instantiating store for column family {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733959178024Cleaning up temporary data from old regions at 1733959178039 (+15 ms)Running coprocessor post-open hooks at 1733959178048 (+9 ms)Region opened successfully at 1733959178049 (+1 ms) 2024-12-11T23:19:38,050 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05., pid=7, masterSystemTime=1733959177986 2024-12-11T23:19:38,053 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05. 2024-12-11T23:19:38,053 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] handler.AssignRegionHandler(153): Opened hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05. 2024-12-11T23:19:38,054 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=301a3ed59652d1a1ff0f40a1ccd90f05, regionState=OPEN, openSeqNum=2, regionLocation=d462fa545078,39401,1733959172007 2024-12-11T23:19:38,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 301a3ed59652d1a1ff0f40a1ccd90f05, server=d462fa545078,39401,1733959172007 because future has completed 2024-12-11T23:19:38,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=7, resume processing ppid=6 2024-12-11T23:19:38,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 301a3ed59652d1a1ff0f40a1ccd90f05, server=d462fa545078,39401,1733959172007 in 231 msec 2024-12-11T23:19:38,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-11T23:19:38,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=301a3ed59652d1a1ff0f40a1ccd90f05, ASSIGN in 399 msec 2024-12-11T23:19:38,073 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T23:19:38,074 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:replication","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733959178073"}]},"ts":"1733959178073"} 2024-12-11T23:19:38,077 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:replication, state=ENABLED in hbase:meta 2024-12-11T23:19:38,079 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T23:19:38,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:replication in 1.2790 sec 2024-12-11T23:19:38,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='1-', locateType=CURRENT is [region=hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05., hostname=d462fa545078,39401,1733959172007, seqNum=2] 2024-12-11T23:19:38,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-11T23:19:38,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:38,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x100172d404c0001, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T23:19:38,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=4, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:38,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=8 2024-12-11T23:19:38,555 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-11T23:19:38,590 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:38,591 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=8}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T23:19:38,591 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=8}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d462fa545078,39401,1733959172007, seqNum=-1] 2024-12-11T23:19:38,591 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=8}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T23:19:38,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58333, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-11T23:19:38,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='1-d462fa545078,39401,1733959172007', locateType=CURRENT is [region=hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05., hostname=d462fa545078,39401,1733959172007, seqNum=2] 2024-12-11T23:19:38,601 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-11T23:19:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-11T23:19:38,607 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:38,612 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-11T23:19:38,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=4 2024-12-11T23:19:38,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 219 msec 2024-12-11T23:19:38,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 1.8740 sec 2024-12-11T23:19:38,618 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:38,620 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x73b6aece to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3f188c20 2024-12-11T23:19:38,645 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:38,645 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T23:19:38,899 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:38,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:19:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T23:19:38,915 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:38,917 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7fdd0d03 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6abcf1b4 2024-12-11T23:19:38,950 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x7fdd0d03 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7fdd0d03 to 127.0.0.1:50115 2024-12-11T23:19:38,950 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T23:19:39,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:39,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=10 2024-12-11T23:19:39,213 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:39,217 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:39,217 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:39,218 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:39,218 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:39,221 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a69c86a to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@178b1749 2024-12-11T23:19:39,222 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T23:19:39,246 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:39,246 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:39,247 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-11T23:19:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-11T23:19:39,250 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:39,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-11T23:19:39,254 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:19:39,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec 2024-12-11T23:19:39,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 348 msec 2024-12-11T23:19:39,263 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:39,264 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4b36911b to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2962e561 2024-12-11T23:19:39,276 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:39,276 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T23:19:39,548 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:39,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:39,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:39,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:39,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-11T23:19:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T23:19:39,567 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:39,569 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x20208661 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@bc6417e 2024-12-11T23:19:39,641 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x20208661 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x20208661 to 127.0.0.1:50115 2024-12-11T23:19:39,642 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:39,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T23:19:39,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=12 2024-12-11T23:19:39,822 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:39,825 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:39,825 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:39,825 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:39,825 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:39,830 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3518cbae to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1fe58053 2024-12-11T23:19:39,831 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:39,848 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:39,848 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:39,848 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-11T23:19:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-11T23:19:39,852 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:39,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-11T23:19:39,857 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-11T23:19:39,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 186 msec 2024-12-11T23:19:39,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 294 msec 2024-12-11T23:19:39,864 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:39,865 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1c2bec11 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@343e4b18 2024-12-11T23:19:39,876 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:39,876 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T23:19:39,887 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:39,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:39,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:39,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:39,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:39,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-11T23:19:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T23:19:39,899 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:39,901 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b73e497 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a235c69 2024-12-11T23:19:39,913 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x1b73e497 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1b73e497 to 127.0.0.1:50115 2024-12-11T23:19:39,913 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:39,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T23:19:40,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=14 2024-12-11T23:19:40,093 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:40,095 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:40,095 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:40,096 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:40,096 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:40,098 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x730cd2c1 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@772856e8 2024-12-11T23:19:40,099 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:40,119 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:40,120 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:40,120 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-11T23:19:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-11T23:19:40,124 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:40,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-11T23:19:40,128 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-11T23:19:40,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-12-11T23:19:40,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 233 msec 2024-12-11T23:19:40,134 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:40,135 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2543c855 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@b2e7f6d 2024-12-11T23:19:40,150 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:40,150 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T23:19:40,218 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:40,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:40,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:40,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:40,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:19:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T23:19:40,232 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:40,234 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x76d0c5ec to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d7f7ff3 2024-12-11T23:19:40,244 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x76d0c5ec {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x76d0c5ec to 127.0.0.1:50115 2024-12-11T23:19:40,245 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:40,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T23:19:40,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=16 2024-12-11T23:19:40,424 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:40,426 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:40,426 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:40,426 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:40,427 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:40,429 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4810bfb1 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e78121f 2024-12-11T23:19:40,430 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:40,451 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:40,451 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:40,451 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-11T23:19:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-11T23:19:40,455 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:40,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-11T23:19:40,459 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:19:40,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-12-11T23:19:40,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 231 msec 2024-12-11T23:19:40,466 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:40,467 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x735a914b to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55d5bbf6 2024-12-11T23:19:40,482 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:40,482 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T23:19:40,547 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:40,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:40,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:40,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-11T23:19:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T23:19:40,555 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:40,558 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3462cf23 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5cf3f6f1 2024-12-11T23:19:40,571 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x3462cf23 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3462cf23 to 127.0.0.1:50115 2024-12-11T23:19:40,571 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:40,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T23:19:40,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=18 2024-12-11T23:19:40,752 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:40,755 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:40,755 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:40,755 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:40,755 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:40,759 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ca9f519 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@a72c4aa 2024-12-11T23:19:40,760 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:40,781 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:40,781 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:40,782 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-11T23:19:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-11T23:19:40,785 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:40,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-11T23:19:40,789 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-11T23:19:40,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec 2024-12-11T23:19:40,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 238 msec 2024-12-11T23:19:40,798 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:40,799 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x34de41ae to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@524bfe05 2024-12-11T23:19:40,813 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:40,813 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:40,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T23:19:40,869 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:40,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:40,873 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:19:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T23:19:40,877 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:40,878 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6fdbca6b to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@240a4ebf 2024-12-11T23:19:40,892 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x6fdbca6b {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6fdbca6b to 127.0.0.1:50115 2024-12-11T23:19:40,892 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:40,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T23:19:41,065 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:replication' 2024-12-11T23:19:41,066 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-11T23:19:41,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=20 2024-12-11T23:19:41,072 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:41,073 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:41,073 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:41,073 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:41,074 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:41,075 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3379baa6 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@25cd859d 2024-12-11T23:19:41,075 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:41,090 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:41,090 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:41,090 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-11T23:19:41,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-11T23:19:41,094 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:41,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-11T23:19:41,098 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:19:41,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec 2024-12-11T23:19:41,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 225 msec 2024-12-11T23:19:41,107 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:41,108 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2de668f0 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7c8da9f 2024-12-11T23:19:41,161 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:41,161 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:41,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T23:19:41,198 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:41,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:41,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:41,208 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:41,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:41,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-11T23:19:41,217 DEBUG [PEWorker-2 {}] procedure.MasterProcedureScheduler(648): PEER '1', shared lock count=0, exclusively locked by procId=22 2024-12-11T23:19:41,217 DEBUG [PEWorker-2 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=21, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:19:41,219 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:41,220 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:41,220 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:41,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:41,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:41,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:41,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:41,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:41,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-11T23:19:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:19:41,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=23 2024-12-11T23:19:41,391 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-11T23:19:41,391 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-11T23:19:41,391 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Replication stream was removed by a user 2024-12-11T23:19:41,391 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:41,392 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:41,395 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x761e40f2 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7b0563bc 2024-12-11T23:19:41,395 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:41,397 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSourceManager(562): Done with the queue 1-d462fa545078,39401,1733959172007 2024-12-11T23:19:41,419 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=23 2024-12-11T23:19:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=23 2024-12-11T23:19:41,423 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:41,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-11T23:19:41,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 187 msec 2024-12-11T23:19:41,427 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-11T23:19:41,429 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-11T23:19:41,439 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-11T23:19:41,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 231 msec 2024-12-11T23:19:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-11T23:19:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:19:41,539 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:41,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-11T23:19:41,543 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:41,548 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=24, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:41,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=24, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec 2024-12-11T23:19:41,550 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-11T23:19:41,650 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='', locateType=CURRENT is [region=hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05., hostname=d462fa545078,39401,1733959172007, seqNum=2] 2024-12-11T23:19:41,727 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=265 (was 214) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x34de41ae java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x4810bfb1-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x735a914b-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x730cd2c1-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x6ca9f519 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x735a914b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_REFRESH_PEER-regionserver/d462fa545078:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187-prefix:d462fa545078,39401,1733959172007.rep java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x4b36911b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x1c2bec11-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2543c855 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x4810bfb1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x73b6aece-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x730cd2c1-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3379baa6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x4b36911b-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0a69c86a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3518cbae-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0a69c86a-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: PacketResponder: BP-1905828385-172.17.0.2-1733959167787:blk_1073741837_1013, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x73b6aece-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x1c2bec11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x73b6aece java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x4810bfb1-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x34de41ae-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3379baa6-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x6ca9f519-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_365039782_22 at /127.0.0.1:40862 [Receiving block BP-1905828385-172.17.0.2-1733959167787:blk_1073741837_1013] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3379baa6-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2de668f0-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3518cbae java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2543c855-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x761e40f2-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x1c2bec11-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x34de41ae-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0a69c86a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2de668f0-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x730cd2c1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x735a914b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x761e40f2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x6ca9f519-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2543c855-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2de668f0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x761e40f2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x4b36911b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3518cbae-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=507 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=101 (was 92) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9066 (was 9104) 2024-12-11T23:19:41,737 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=265, OpenFileDescriptor=507, MaxFileDescriptor=1048576, SystemLoadAverage=101, ProcessCount=11, AvailableMemoryMB=9065 2024-12-11T23:19:41,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T23:19:41,744 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:41,746 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x17f9c12e to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6dea7d37 2024-12-11T23:19:41,771 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x17f9c12e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x17f9c12e to 127.0.0.1:50115 2024-12-11T23:19:41,772 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:41,801 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T23:19:41,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:19:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T23:19:41,848 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-11T23:19:41,849 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-11T23:19:41,851 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_replication 2024-12-11T23:19:41,851 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_replication Metrics about Tables on a single HBase RegionServer 2024-12-11T23:19:41,852 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T23:19:41,852 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-11T23:19:41,853 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-11T23:19:41,853 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-11T23:19:41,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=26 2024-12-11T23:19:41,995 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-11T23:19:42,022 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:42,025 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-11T23:19:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-11T23:19:42,028 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:42,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-11T23:19:42,031 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-11T23:19:42,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 187 msec 2024-12-11T23:19:42,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 292 msec 2024-12-11T23:19:42,039 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:42,040 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58990532 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6e862abe 2024-12-11T23:19:42,055 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:42,055 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T23:19:42,057 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:42,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:19:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-11T23:19:42,063 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:42,064 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3b4958c6 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d942bf3 2024-12-11T23:19:42,076 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x3b4958c6 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3b4958c6 to 127.0.0.1:50115 2024-12-11T23:19:42,076 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:42,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-11T23:19:42,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=28 2024-12-11T23:19:42,255 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:42,258 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:42,258 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:42,259 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:42,259 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:42,263 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0fe865dc to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1ed84483 2024-12-11T23:19:42,263 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:42,282 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:42,282 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:42,282 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-11T23:19:42,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-11T23:19:42,286 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:42,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-11T23:19:42,289 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:19:42,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 187 msec 2024-12-11T23:19:42,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 231 msec 2024-12-11T23:19:42,300 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:42,301 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x45665e3f to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@13ed5162 2024-12-11T23:19:42,313 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:42,313 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:42,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:19:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-11T23:19:42,377 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:42,379 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-11T23:19:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-11T23:19:42,386 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:42,389 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x44932d39 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7afe9416 2024-12-11T23:19:42,402 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x44932d39 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x44932d39 to 127.0.0.1:50115 2024-12-11T23:19:42,402 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:42,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-11T23:19:42,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=30 2024-12-11T23:19:42,580 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:42,581 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:42,581 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:42,582 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:42,582 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:42,584 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x37052b90 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5ecb978e 2024-12-11T23:19:42,584 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:42,608 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:42,609 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:42,609 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-12-11T23:19:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-12-11T23:19:42,612 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:42,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-11T23:19:42,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-12-11T23:19:42,615 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-11T23:19:42,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 235 msec 2024-12-11T23:19:42,623 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:42,624 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x03e0174b to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7968139e 2024-12-11T23:19:42,633 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:42,634 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-11T23:19:42,706 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:42,708 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-12-11T23:19:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-11T23:19:42,712 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:42,714 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3fe85ea1 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d9a25eb 2024-12-11T23:19:42,729 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x3fe85ea1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3fe85ea1 to 127.0.0.1:50115 2024-12-11T23:19:42,729 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:42,730 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:42,733 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=31, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:42,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=31, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=26 msec 2024-12-11T23:19:42,736 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-11T23:19:42,817 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-12-11T23:19:42,819 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-11T23:19:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-11T23:19:42,823 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:42,825 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x23999834 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1c7c403f 2024-12-11T23:19:42,840 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x23999834 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x23999834 to 127.0.0.1:50115 2024-12-11T23:19:42,841 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:42,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-11T23:19:43,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=33 2024-12-11T23:19:43,019 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:43,021 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:43,021 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:43,021 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:43,021 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:43,023 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0db74723 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a3b0a44 2024-12-11T23:19:43,023 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:43,038 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:43,039 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:43,039 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-12-11T23:19:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-12-11T23:19:43,042 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:43,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-11T23:19:43,046 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-11T23:19:43,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec 2024-12-11T23:19:43,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 227 msec 2024-12-11T23:19:43,053 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:43,054 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a52a7ce to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2b97f13 2024-12-11T23:19:43,065 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:43,065 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-11T23:19:43,137 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:43,138 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-11T23:19:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-11T23:19:43,141 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:43,142 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x34caa818 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@688721a0 2024-12-11T23:19:43,155 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x34caa818 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x34caa818 to 127.0.0.1:50115 2024-12-11T23:19:43,155 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:43,156 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:43,157 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=34, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:43,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=34, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=20 msec 2024-12-11T23:19:43,159 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-11T23:19:43,248 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-12-11T23:19:43,249 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-11T23:19:43,258 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:43,259 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:43,260 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:43,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:43,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:43,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:43,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:43,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:43,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-11T23:19:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:19:43,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=36 2024-12-11T23:19:43,432 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-11T23:19:43,432 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-11T23:19:43,432 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Replication stream was removed by a user 2024-12-11T23:19:43,432 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:43,433 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:43,435 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7730277e to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@20ffce00 2024-12-11T23:19:43,436 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:43,437 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSourceManager(562): Done with the queue 1-d462fa545078,39401,1733959172007 2024-12-11T23:19:43,442 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=36 2024-12-11T23:19:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=36 2024-12-11T23:19:43,446 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:43,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-11T23:19:43,450 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-11T23:19:43,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 169 msec 2024-12-11T23:19:43,450 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-11T23:19:43,457 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-11T23:19:43,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 207 msec 2024-12-11T23:19:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-11T23:19:43,570 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:43,571 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:43,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-11T23:19:43,577 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:43,579 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=37, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:43,581 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=37, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec 2024-12-11T23:19:43,581 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-11T23:19:43,691 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-11T23:19:43,692 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T23:19:43,696 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:43,698 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=38, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:43,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=38, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-11T23:19:43,700 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T23:19:43,809 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:43,829 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=290 (was 265) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0fe865dc java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x7730277e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0fe865dc-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x37052b90-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x37052b90-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x7730277e-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0fe865dc-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x03e0174b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x45665e3f-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x58990532-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x58990532 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3a52a7ce java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3a52a7ce-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0db74723 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3a52a7ce-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x58990532-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x45665e3f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x37052b90 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x7730277e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0db74723-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x45665e3f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x03e0174b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0db74723-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x03e0174b-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=540 (was 507) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=101 (was 101), ProcessCount=11 (was 11), AvailableMemoryMB=9057 (was 9065) 2024-12-11T23:19:43,839 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=290, OpenFileDescriptor=540, MaxFileDescriptor=1048576, SystemLoadAverage=101, ProcessCount=11, AvailableMemoryMB=9057 2024-12-11T23:19:43,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-11T23:19:43,844 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:43,846 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x675d6dfd to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@46094e1c 2024-12-11T23:19:43,903 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x675d6dfd {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x675d6dfd to 127.0.0.1:50115 2024-12-11T23:19:43,903 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:43,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-11T23:19:44,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=40 2024-12-11T23:19:44,095 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-11T23:19:44,129 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:44,132 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-12-11T23:19:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-12-11T23:19:44,135 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:44,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-11T23:19:44,137 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-11T23:19:44,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 198 msec 2024-12-11T23:19:44,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 297 msec 2024-12-11T23:19:44,146 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:44,147 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b7f1575 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1cb1f58d 2024-12-11T23:19:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-11T23:19:44,156 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:44,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41 2024-12-11T23:19:44,160 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:44,160 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:44,160 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:44,162 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4a44acd1 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b6f458b 2024-12-11T23:19:44,177 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x4a44acd1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4a44acd1 to 127.0.0.1:50115 2024-12-11T23:19:44,177 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:44,177 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:44,180 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=41, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:44,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=41, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=23 msec 2024-12-11T23:19:44,182 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41 2024-12-11T23:19:44,267 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-12-11T23:19:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-11T23:19:44,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T23:19:44,280 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:44,282 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=42, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:44,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=42, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-11T23:19:44,283 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T23:19:44,389 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-11T23:19:44,393 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=43, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-12-11T23:19:44,399 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test2' 2024-12-11T23:19:44,401 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0881bccf to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@44108a7 2024-12-11T23:19:44,412 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x0881bccf {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0881bccf to 127.0.0.1:50115 2024-12-11T23:19:44,413 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test2 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:44,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-12-11T23:19:44,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=44 2024-12-11T23:19:44,631 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=ADD_PEER 2024-12-11T23:19:44,669 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.ReplicationSource(231): queueId=2-d462fa545078,39401,1733959172007, ReplicationSource: 2, currentBandwidth=0 2024-12-11T23:19:44,672 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=44 2024-12-11T23:19:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=44 2024-12-11T23:19:44,675 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for ADD on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:44,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-12-11T23:19:44,679 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 2, config clusterKey=hbase+zk://127.0.0.1:50115/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-11T23:19:44,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 199 msec 2024-12-11T23:19:44,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 285 msec 2024-12-11T23:19:44,686 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,2-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test2' 2024-12-11T23:19:44,687 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,2-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a3942de to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@417195ac 2024-12-11T23:19:44,707 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,2-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:44,708 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,2-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-12-11T23:19:44,718 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-12-11T23:19:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-11T23:19:44,721 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=45, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45 2024-12-11T23:19:44,728 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:44,728 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:44,729 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:44,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:44,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:44,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:44,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:44,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:44,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45 2024-12-11T23:19:44,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=46 2024-12-11T23:19:44,899 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-11T23:19:44,899 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-11T23:19:44,899 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Replication stream was removed by a user 2024-12-11T23:19:44,899 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:44,899 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:44,900 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x05497410 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5dd66b53 2024-12-11T23:19:44,901 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:44,901 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSourceManager(562): Done with the queue 1-d462fa545078,39401,1733959172007 2024-12-11T23:19:44,904 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46 2024-12-11T23:19:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=46 2024-12-11T23:19:44,907 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:44,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=45 2024-12-11T23:19:44,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=45, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 162 msec 2024-12-11T23:19:44,909 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-11T23:19:44,909 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-11T23:19:44,916 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-11T23:19:44,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 195 msec 2024-12-11T23:19:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45 2024-12-11T23:19:45,047 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:45,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-11T23:19:45,049 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:45,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:45,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-11T23:19:45,054 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/sync-rep-state 2024-12-11T23:19:45,054 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-12-11T23:19:45,054 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/new-sync-rep-state 2024-12-11T23:19:45,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/new-sync-rep-state 2024-12-11T23:19:45,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-11T23:19:45,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-12-11T23:19:45,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/sync-rep-state 2024-12-11T23:19:45,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-11T23:19:45,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-11T23:19:45,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=48 2024-12-11T23:19:45,220 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-12-11T23:19:45,220 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 2: 0 2024-12-11T23:19:45,220 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSource(687): peerId=2, Closing source 2-d462fa545078,39401,1733959172007 because: Replication stream was removed by a user 2024-12-11T23:19:45,220 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,2-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=2, Interrupted while sleeping between retries 2024-12-11T23:19:45,220 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,2-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test2' 2024-12-11T23:19:45,221 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,2-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x323c9517 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2a2dc9eb 2024-12-11T23:19:45,221 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,2-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:45,222 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(562): Done with the queue 2-d462fa545078,39401,1733959172007 2024-12-11T23:19:45,225 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-12-11T23:19:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-12-11T23:19:45,228 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for REMOVE on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:45,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-12-11T23:19:45,230 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 2, move on 2024-12-11T23:19:45,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 162 msec 2024-12-11T23:19:45,231 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 2, move on 2024-12-11T23:19:45,237 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 2 2024-12-11T23:19:45,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 188 msec 2024-12-11T23:19:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-11T23:19:45,369 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-12-11T23:19:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-11T23:19:45,373 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:45,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-12-11T23:19:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:19:45,377 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:45,379 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=49, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:45,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=49, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-11T23:19:45,381 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-12-11T23:19:45,487 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-11T23:19:45,488 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-11T23:19:45,492 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:45,493 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=50, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:45,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=50, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-11T23:19:45,495 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-11T23:19:45,597 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:45,611 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=302 (was 290) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2b7f1575-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3a3942de-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x05497410 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x323c9517 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x05497410-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x05497410-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3a3942de-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x323c9517-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2b7f1575 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3a3942de java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x323c9517-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2b7f1575-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=555 (was 540) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=141 (was 101) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9008 (was 9057) 2024-12-11T23:19:45,620 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=302, OpenFileDescriptor=555, MaxFileDescriptor=1048576, SystemLoadAverage=141, ProcessCount=11, AvailableMemoryMB=9006 2024-12-11T23:19:45,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=51, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-12-11T23:19:45,624 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] ... 11 more 2024-12-11T23:19:45,625 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=51, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:45,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=51, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=5 msec 2024-12-11T23:19:45,627 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-12-11T23:19:45,727 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-12-11T23:19:45,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-11T23:19:45,735 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:45,736 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=52, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:45,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=52, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-11T23:19:45,739 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-11T23:19:45,837 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-11T23:19:45,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-11T23:19:45,840 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:45,842 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=53, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:45,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=53, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-11T23:19:45,843 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-11T23:19:45,948 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:45,967 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=302 (was 302), OpenFileDescriptor=555 (was 555), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=141 (was 141), ProcessCount=11 (was 11), AvailableMemoryMB=8987 (was 9006) 2024-12-11T23:19:45,976 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=302, OpenFileDescriptor=555, MaxFileDescriptor=1048576, SystemLoadAverage=141, ProcessCount=11, AvailableMemoryMB=8987 2024-12-11T23:19:45,977 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-11T23:19:45,980 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-11T23:19:45,982 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=54, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:45,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=54, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=5 msec 2024-12-11T23:19:45,983 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-11T23:19:46,088 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-12-11T23:19:46,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-11T23:19:46,093 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:46,096 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=55, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:46,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=55, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-11T23:19:46,098 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-11T23:19:46,199 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-11T23:19:46,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T23:19:46,206 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:46,207 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=56, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:46,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=56, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-11T23:19:46,209 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T23:19:46,318 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:46,342 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=302 (was 302), OpenFileDescriptor=555 (was 555), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=141 (was 141), ProcessCount=11 (was 11), AvailableMemoryMB=8989 (was 8987) - AvailableMemoryMB LEAK? - 2024-12-11T23:19:46,354 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=302, OpenFileDescriptor=555, MaxFileDescriptor=1048576, SystemLoadAverage=141, ProcessCount=11, AvailableMemoryMB=8989 2024-12-11T23:19:46,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-11T23:19:46,359 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:46,361 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7503c0ea to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6692fb5b 2024-12-11T23:19:46,430 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x7503c0ea {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7503c0ea to 127.0.0.1:50115 2024-12-11T23:19:46,431 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-11T23:19:46,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:46,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=58 2024-12-11T23:19:46,625 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-11T23:19:46,658 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:46,661 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=58 2024-12-11T23:19:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=58 2024-12-11T23:19:46,664 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:46,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-11T23:19:46,667 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-11T23:19:46,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec 2024-12-11T23:19:46,668 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 312 msec 2024-12-11T23:19:46,676 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-11T23:19:46,677 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x57a18bac to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@126c1c7 2024-12-11T23:19:46,677 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:46,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:19:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-11T23:19:46,681 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:46,682 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x006b71a2 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@13669b3 2024-12-11T23:19:46,717 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:46,717 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:46,717 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x006b71a2 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x006b71a2 to 127.0.0.1:50115 2024-12-11T23:19:46,717 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:46,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-11T23:19:46,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=60 2024-12-11T23:19:46,897 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:46,900 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T23:19:46,901 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:46,901 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:46,901 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:46,902 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:46,903 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59b55eed to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55e5dd34 2024-12-11T23:19:46,903 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:46,920 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:46,921 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:46,921 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-12-11T23:19:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=60 2024-12-11T23:19:46,925 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:46,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-11T23:19:46,929 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:19:46,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 185 msec 2024-12-11T23:19:46,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 251 msec 2024-12-11T23:19:46,938 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:46,940 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x52af179f to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34e89538 2024-12-11T23:19:46,949 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:46,949 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-11T23:19:46,997 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:46,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-11T23:19:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-11T23:19:47,002 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:47,003 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5b96ddbe to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4cb3881d 2024-12-11T23:19:47,019 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x5b96ddbe {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5b96ddbe to 127.0.0.1:50115 2024-12-11T23:19:47,019 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:47,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-11T23:19:47,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=62 2024-12-11T23:19:47,194 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:47,197 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:47,197 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:47,197 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:47,198 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:47,202 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x339aa5aa to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@715777b1 2024-12-11T23:19:47,202 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:47,220 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:47,220 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:47,220 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-12-11T23:19:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-12-11T23:19:47,223 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:47,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-11T23:19:47,226 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-11T23:19:47,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec 2024-12-11T23:19:47,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 228 msec 2024-12-11T23:19:47,236 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:47,237 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1732b7ea to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@df40d00 2024-12-11T23:19:47,250 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:47,250 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-11T23:19:47,318 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:47,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:47,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-11T23:19:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-11T23:19:47,328 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:47,330 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x20653480 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@56e6082b 2024-12-11T23:19:47,345 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x20653480 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x20653480 to 127.0.0.1:50115 2024-12-11T23:19:47,345 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:47,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-11T23:19:47,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=64 2024-12-11T23:19:47,523 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:47,526 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:47,527 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:47,527 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:47,527 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:47,531 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3b121ec8 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4f75b132 2024-12-11T23:19:47,531 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:47,549 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:47,549 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:47,550 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=64 2024-12-11T23:19:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=64 2024-12-11T23:19:47,553 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:47,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-11T23:19:47,556 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-11T23:19:47,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 186 msec 2024-12-11T23:19:47,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 235 msec 2024-12-11T23:19:47,564 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:47,565 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6cab901a to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@43c0b20f 2024-12-11T23:19:47,576 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:47,576 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-11T23:19:47,648 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:47,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:47,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-11T23:19:47,656 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:47,657 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:47,658 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:47,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:47,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:47,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:47,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:47,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:47,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-11T23:19:47,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=66 2024-12-11T23:19:47,826 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-11T23:19:47,826 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-11T23:19:47,826 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Replication stream was removed by a user 2024-12-11T23:19:47,827 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:47,827 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:47,831 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x03f58343 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a007808 2024-12-11T23:19:47,831 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:47,832 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(562): Done with the queue 1-d462fa545078,39401,1733959172007 2024-12-11T23:19:47,836 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-12-11T23:19:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-12-11T23:19:47,840 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:47,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-12-11T23:19:47,843 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-11T23:19:47,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 168 msec 2024-12-11T23:19:47,843 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-11T23:19:47,849 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-11T23:19:47,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 199 msec 2024-12-11T23:19:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-11T23:19:47,967 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:47,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-11T23:19:47,970 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:47,972 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=67, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:47,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=67, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-11T23:19:47,973 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-11T23:19:48,079 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-11T23:19:48,080 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-11T23:19:48,085 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:48,087 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=68, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:48,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=68, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec 2024-12-11T23:19:48,089 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-11T23:19:48,197 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:48,211 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=322 (was 302) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x57a18bac-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x1732b7ea-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3b121ec8-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x339aa5aa java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x57a18bac java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3b121ec8-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x03f58343-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x52af179f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x6cab901a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x59b55eed-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x59b55eed-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x339aa5aa-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x03f58343 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x6cab901a-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x03f58343-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x6cab901a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x59b55eed java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x52af179f-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x339aa5aa-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x52af179f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3b121ec8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x57a18bac-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x1732b7ea-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x1732b7ea java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=583 (was 555) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=141 (was 141), ProcessCount=11 (was 11), AvailableMemoryMB=8966 (was 8989) 2024-12-11T23:19:48,222 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=322, OpenFileDescriptor=583, MaxFileDescriptor=1048576, SystemLoadAverage=141, ProcessCount=11, AvailableMemoryMB=8964 2024-12-11T23:19:48,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=69, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-12-11T23:19:48,226 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:48,227 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3805dcd7 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42004c01 2024-12-11T23:19:48,281 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x3805dcd7 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3805dcd7 to 127.0.0.1:50115 2024-12-11T23:19:48,282 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:48,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-12-11T23:19:48,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=70 2024-12-11T23:19:48,468 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-11T23:19:48,498 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:48,501 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=70 2024-12-11T23:19:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=70 2024-12-11T23:19:48,504 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:48,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-12-11T23:19:48,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec 2024-12-11T23:19:48,506 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-11T23:19:48,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 283 msec 2024-12-11T23:19:48,515 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:48,516 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d0c31ff to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@335ebda2 2024-12-11T23:19:48,528 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:48,528 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-12-11T23:19:48,547 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:48,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:19:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=71, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-12-11T23:19:48,553 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:48,555 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4a54e3f6 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b926bf7 2024-12-11T23:19:48,602 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x4a54e3f6 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4a54e3f6 to 127.0.0.1:50115 2024-12-11T23:19:48,602 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:48,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-12-11T23:19:48,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=72 2024-12-11T23:19:48,789 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:48,792 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:48,793 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:48,793 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:48,793 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:48,798 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1507cc71 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@67bdd5e 2024-12-11T23:19:48,799 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:48,818 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:48,818 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:48,818 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72 2024-12-11T23:19:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=72 2024-12-11T23:19:48,821 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:48,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=71 2024-12-11T23:19:48,824 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:19:48,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 187 msec 2024-12-11T23:19:48,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 276 msec 2024-12-11T23:19:48,833 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:48,834 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3303870e to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@59d3be32 2024-12-11T23:19:48,844 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:48,844 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:48,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-12-11T23:19:48,867 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:48,867 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:48,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-11T23:19:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=73, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-12-11T23:19:48,872 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:48,873 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x39e680b1 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@167aef2a 2024-12-11T23:19:48,886 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x39e680b1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x39e680b1 to 127.0.0.1:50115 2024-12-11T23:19:48,886 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:48,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-12-11T23:19:49,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=74 2024-12-11T23:19:49,064 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:49,067 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:49,067 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:49,067 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:49,068 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:49,071 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59b483b6 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@587e242 2024-12-11T23:19:49,071 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:49,093 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:49,094 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:49,094 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74 2024-12-11T23:19:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=74 2024-12-11T23:19:49,097 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:49,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-12-11T23:19:49,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec 2024-12-11T23:19:49,099 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-11T23:19:49,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 231 msec 2024-12-11T23:19:49,108 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:49,109 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x182271ee to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4c87b716 2024-12-11T23:19:49,123 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:49,124 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-12-11T23:19:49,188 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:49,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:49,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:49,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-11T23:19:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=75, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:49,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-11T23:19:49,201 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:49,204 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ce9c197 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@52050f40 2024-12-11T23:19:49,233 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x7ce9c197 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ce9c197 to 127.0.0.1:50115 2024-12-11T23:19:49,233 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:49,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-11T23:19:49,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:19:49,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=76 2024-12-11T23:19:49,409 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:49,410 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:49,410 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:49,410 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:49,410 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:49,412 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2ec3b04d to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4a057973 2024-12-11T23:19:49,413 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:49,429 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:49,429 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:49,430 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76 2024-12-11T23:19:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=76 2024-12-11T23:19:49,433 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:49,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-12-11T23:19:49,436 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-11T23:19:49,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 178 msec 2024-12-11T23:19:49,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 242 msec 2024-12-11T23:19:49,444 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:49,445 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3b144df5 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@69398b8 2024-12-11T23:19:49,454 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:49,454 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-11T23:19:49,517 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:49,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:49,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:49,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-11T23:19:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=77, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-11T23:19:49,522 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:49,523 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x459148d8 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1409faf8 2024-12-11T23:19:49,534 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x459148d8 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x459148d8 to 127.0.0.1:50115 2024-12-11T23:19:49,534 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:49,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-11T23:19:49,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=78 2024-12-11T23:19:49,710 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:49,713 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:49,713 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:49,713 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:49,713 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:49,716 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a9c6ed6 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@66c13ef 2024-12-11T23:19:49,717 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:49,736 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:49,737 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:49,737 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78 2024-12-11T23:19:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=78 2024-12-11T23:19:49,740 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:49,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-12-11T23:19:49,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 185 msec 2024-12-11T23:19:49,742 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-11T23:19:49,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 223 msec 2024-12-11T23:19:49,750 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:49,752 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4bdc13ef to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@80f031b 2024-12-11T23:19:49,765 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:49,766 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-11T23:19:49,838 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:49,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:49,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:49,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-11T23:19:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-11T23:19:49,850 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:49,850 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x10ba5644 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@71ad9fa3 2024-12-11T23:19:49,865 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x10ba5644 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x10ba5644 to 127.0.0.1:50115 2024-12-11T23:19:49,865 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:49,886 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-11T23:19:50,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=80 2024-12-11T23:19:50,042 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:50,045 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:50,045 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:50,045 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:50,046 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:50,048 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5df29fae to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4c74eb79 2024-12-11T23:19:50,048 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:50,063 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:50,063 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:50,064 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80 2024-12-11T23:19:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=80 2024-12-11T23:19:50,066 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:50,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-11T23:19:50,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec 2024-12-11T23:19:50,069 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-11T23:19:50,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 225 msec 2024-12-11T23:19:50,078 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:50,079 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a7f1dae to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@79cdf853 2024-12-11T23:19:50,092 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:50,092 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-11T23:19:50,168 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:50,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:50,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:50,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-11T23:19:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-11T23:19:50,179 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:50,180 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0049f260 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d21516f 2024-12-11T23:19:50,191 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x0049f260 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0049f260 to 127.0.0.1:50115 2024-12-11T23:19:50,191 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:50,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-11T23:19:50,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=82 2024-12-11T23:19:50,368 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:50,369 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:50,369 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:50,369 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:50,369 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:50,371 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5baf0f43 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4cd1d9e0 2024-12-11T23:19:50,371 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:50,393 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:50,394 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:50,394 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-12-11T23:19:50,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=82 2024-12-11T23:19:50,397 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:50,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-11T23:19:50,399 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-11T23:19:50,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec 2024-12-11T23:19:50,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 224 msec 2024-12-11T23:19:50,408 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:50,409 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ff9e568 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5797b344 2024-12-11T23:19:50,423 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:50,423 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-11T23:19:50,498 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:50,499 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:50,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-11T23:19:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-11T23:19:50,507 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:50,509 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4dca9fcb to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3536329 2024-12-11T23:19:50,524 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x4dca9fcb {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4dca9fcb to 127.0.0.1:50115 2024-12-11T23:19:50,524 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:50,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-11T23:19:50,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=84 2024-12-11T23:19:50,700 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:50,702 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-12-11T23:19:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=84 2024-12-11T23:19:50,704 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:50,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-11T23:19:50,707 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-11T23:19:50,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec 2024-12-11T23:19:50,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 206 msec 2024-12-11T23:19:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-11T23:19:50,827 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:50,828 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:50,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:50,830 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-11T23:19:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-11T23:19:50,834 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:50,835 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x45b9aad0 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@656ff266 2024-12-11T23:19:50,843 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x45b9aad0 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x45b9aad0 to 127.0.0.1:50115 2024-12-11T23:19:50,844 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:50,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-11T23:19:51,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=86 2024-12-11T23:19:51,018 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:51,020 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:51,020 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:51,020 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:51,020 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:51,022 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x456005b4 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@482bc577 2024-12-11T23:19:51,022 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:51,038 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:51,038 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:51,038 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-12-11T23:19:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-12-11T23:19:51,041 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:51,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-11T23:19:51,044 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-11T23:19:51,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 176 msec 2024-12-11T23:19:51,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 214 msec 2024-12-11T23:19:51,053 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:51,054 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x186f86f5 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7789316f 2024-12-11T23:19:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-11T23:19:51,147 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:51,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:51,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-11T23:19:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-11T23:19:51,153 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:51,154 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0f2e2c17 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4a6b536a 2024-12-11T23:19:51,180 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:51,180 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:51,180 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x0f2e2c17 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0f2e2c17 to 127.0.0.1:50115 2024-12-11T23:19:51,181 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:51,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-11T23:19:51,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=88 2024-12-11T23:19:51,355 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:51,357 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:51,357 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:51,357 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:51,357 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:51,358 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5e1b5946 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4800ec26 2024-12-11T23:19:51,358 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:51,375 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:51,375 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:51,375 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-12-11T23:19:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-12-11T23:19:51,378 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:51,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-11T23:19:51,381 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-11T23:19:51,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 176 msec 2024-12-11T23:19:51,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 232 msec 2024-12-11T23:19:51,390 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:51,391 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d4f7d2b to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@770529e5 2024-12-11T23:19:51,401 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:51,402 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-11T23:19:51,467 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:51,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:51,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-11T23:19:51,473 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:51,474 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:51,474 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:51,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:51,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:51,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:51,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:51,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:51,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-11T23:19:51,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=90 2024-12-11T23:19:51,640 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-11T23:19:51,640 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-11T23:19:51,640 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Replication stream was removed by a user 2024-12-11T23:19:51,640 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:51,640 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:51,641 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c55c34c to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ed2ae0b 2024-12-11T23:19:51,642 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:51,642 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(562): Done with the queue 1-d462fa545078,39401,1733959172007 2024-12-11T23:19:51,648 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-11T23:19:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-11T23:19:51,651 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:51,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-11T23:19:51,654 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-11T23:19:51,654 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-11T23:19:51,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 165 msec 2024-12-11T23:19:51,664 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-11T23:19:51,666 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 195 msec 2024-12-11T23:19:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-11T23:19:51,787 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:51,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-11T23:19:51,790 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:51,791 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=91, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:51,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=91, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-11T23:19:51,792 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-11T23:19:51,898 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-11T23:19:51,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-11T23:19:51,900 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:51,902 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=92, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:51,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=92, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-11T23:19:51,903 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-11T23:19:52,007 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:52,023 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=376 (was 322) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x5e1b5946-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x186f86f5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x59b483b6-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x456005b4-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3b144df5-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x182271ee-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x5baf0f43 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x7ff9e568-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x456005b4-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x186f86f5-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x5baf0f43-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x5df29fae-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2a9c6ed6-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x182271ee-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3d0c31ff java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x182271ee java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x7ff9e568-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3303870e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x4bdc13ef-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3303870e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3c55c34c-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2ec3b04d-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0a7f1dae java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3d0c31ff-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2ec3b04d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x59b483b6-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0d4f7d2b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0a7f1dae-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x4bdc13ef-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x456005b4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2ec3b04d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x7ff9e568 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3303870e-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x1507cc71-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x5df29fae java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0d4f7d2b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0a7f1dae-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2a9c6ed6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3d0c31ff-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x0d4f7d2b-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x1507cc71 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x5e1b5946 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x1507cc71-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3c55c34c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x5df29fae-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x2a9c6ed6-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3b144df5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3c55c34c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x5e1b5946-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x59b483b6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x4bdc13ef java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x186f86f5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3b144df5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x5baf0f43-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=653 (was 583) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=145 (was 141) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8824 (was 8964) 2024-12-11T23:19:52,031 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=376, OpenFileDescriptor=653, MaxFileDescriptor=1048576, SystemLoadAverage=145, ProcessCount=11, AvailableMemoryMB=8822 2024-12-11T23:19:52,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-11T23:19:52,035 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:52,036 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x14b731e4 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ab1bc5 2024-12-11T23:19:52,081 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x14b731e4 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x14b731e4 to 127.0.0.1:50115 2024-12-11T23:19:52,081 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:52,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-11T23:19:52,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=94 2024-12-11T23:19:52,266 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-11T23:19:52,292 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:52,295 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-11T23:19:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-11T23:19:52,297 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:52,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-11T23:19:52,299 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-11T23:19:52,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec 2024-12-11T23:19:52,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 266 msec 2024-12-11T23:19:52,310 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:52,311 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5af69c10 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6d1179a8 2024-12-11T23:19:52,323 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:52,324 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-11T23:19:52,357 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-11T23:19:52,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3992): Client=jenkins//172.17.0.2 disable replication peer, id=1 2024-12-11T23:19:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-12-11T23:19:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-11T23:19:52,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-11T23:19:52,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=96 2024-12-11T23:19:52,530 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-12-11T23:19:52,531 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:19:52,531 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:19:52,531 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:52,531 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:52,533 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6e04bf1b to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@90ddda6 2024-12-11T23:19:52,533 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:52,548 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:52,548 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:19:52,548 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-11T23:19:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-11T23:19:52,551 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for DISABLE on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:52,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-11T23:19:52,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 174 msec 2024-12-11T23:19:52,553 INFO [PEWorker-2 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-12-11T23:19:52,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 191 msec 2024-12-11T23:19:52,562 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:52,563 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7b00c8de to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@20b47a09 2024-12-11T23:19:52,576 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:52,576 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-11T23:19:52,687 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-11T23:19:52,688 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T23:19:52,692 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:52,693 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:52,693 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:52,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:52,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:52,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:52,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:52,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:52,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T23:19:52,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=98 2024-12-11T23:19:52,860 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-11T23:19:52,861 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-11T23:19:52,861 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Replication stream was removed by a user 2024-12-11T23:19:52,861 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:52,861 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:52,862 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b10a65d to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@70627bde 2024-12-11T23:19:52,862 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:52,863 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(562): Done with the queue 1-d462fa545078,39401,1733959172007 2024-12-11T23:19:52,865 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-11T23:19:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-11T23:19:52,867 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:52,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-11T23:19:52,869 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-11T23:19:52,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec 2024-12-11T23:19:52,869 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-11T23:19:52,875 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-11T23:19:52,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 187 msec 2024-12-11T23:19:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T23:19:53,007 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:53,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T23:19:53,010 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:53,011 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=99, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:53,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=99, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-11T23:19:53,013 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T23:19:53,117 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-11T23:19:53,117 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-12-11T23:19:53,119 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:53,120 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=100, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:53,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=100, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-11T23:19:53,121 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:53,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-12-11T23:19:53,227 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:53,241 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=387 (was 376) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x6e04bf1b-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x6b10a65d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x5af69c10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x5af69c10-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x6e04bf1b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x7b00c8de-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x6b10a65d-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x6b10a65d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x7b00c8de-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x7b00c8de java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x6e04bf1b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x5af69c10-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=669 (was 653) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=145 (was 145), ProcessCount=11 (was 11), AvailableMemoryMB=8808 (was 8822) 2024-12-11T23:19:53,250 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=387, OpenFileDescriptor=669, MaxFileDescriptor=1048576, SystemLoadAverage=145, ProcessCount=11, AvailableMemoryMB=8807 2024-12-11T23:19:53,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=dummy_peer 2024-12-11T23:19:53,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:53,254 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:53,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T23:19:53,255 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=101, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:53,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=101, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-11T23:19:53,255 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:53,334 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T23:19:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T23:19:53,357 INFO [Time-limited test {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-12-11T23:19:53,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=102, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:53,362 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:53,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-11T23:19:53,364 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=102, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:53,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=102, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-11T23:19:53,365 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-11T23:19:53,467 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-11T23:19:53,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T23:19:53,471 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:53,472 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=103, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:53,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=103, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-11T23:19:53,474 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T23:19:53,578 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:53,601 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=388 (was 387) - Thread LEAK? -, OpenFileDescriptor=669 (was 669), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=145 (was 145), ProcessCount=11 (was 11), AvailableMemoryMB=8801 (was 8807) 2024-12-11T23:19:53,613 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=388, OpenFileDescriptor=669, MaxFileDescriptor=1048576, SystemLoadAverage=145, ProcessCount=11, AvailableMemoryMB=8800 2024-12-11T23:19:53,615 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-11T23:19:53,619 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:53,620 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x53391e70 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@22b1c332 2024-12-11T23:19:53,670 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x53391e70 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x53391e70 to 127.0.0.1:50115 2024-12-11T23:19:53,671 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:53,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-11T23:19:53,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=105 2024-12-11T23:19:53,855 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-11T23:19:53,880 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:53,883 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=105 2024-12-11T23:19:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=105 2024-12-11T23:19:53,885 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:53,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-12-11T23:19:53,888 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-11T23:19:53,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec 2024-12-11T23:19:53,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 273 msec 2024-12-11T23:19:53,899 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:53,900 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b278bf3 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3834259d 2024-12-11T23:19:53,928 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:53,928 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-11T23:19:53,937 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-11T23:19:53,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-11T23:19:53,941 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:53,942 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:53,942 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:53,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:53,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:53,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:53,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:53,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:53,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-11T23:19:54,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=107 2024-12-11T23:19:54,109 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-11T23:19:54,109 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-11T23:19:54,109 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Replication stream was removed by a user 2024-12-11T23:19:54,109 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:54,109 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:54,110 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x24fd53e0 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@637d806c 2024-12-11T23:19:54,111 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:54,111 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSourceManager(562): Done with the queue 1-d462fa545078,39401,1733959172007 2024-12-11T23:19:54,113 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107 2024-12-11T23:19:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=107 2024-12-11T23:19:54,116 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:54,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-12-11T23:19:54,117 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-11T23:19:54,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 160 msec 2024-12-11T23:19:54,118 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-11T23:19:54,123 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-11T23:19:54,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 185 msec 2024-12-11T23:19:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-11T23:19:54,257 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:54,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-11T23:19:54,260 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:54,260 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=108, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:54,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=108, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-11T23:19:54,262 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-11T23:19:54,367 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-11T23:19:54,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-11T23:19:54,369 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:54,370 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=109, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:54,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=109, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-11T23:19:54,371 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-11T23:19:54,478 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:54,493 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=394 (was 388) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x1b278bf3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x24fd53e0-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x24fd53e0-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x24fd53e0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x1b278bf3-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x1b278bf3-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=677 (was 669) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=145 (was 145), ProcessCount=11 (was 11), AvailableMemoryMB=8780 (was 8800) 2024-12-11T23:19:54,506 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=394, OpenFileDescriptor=677, MaxFileDescriptor=1048576, SystemLoadAverage=145, ProcessCount=11, AvailableMemoryMB=8779 2024-12-11T23:19:54,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] ipc.CallRunner(138): callId: 313 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.2:41904 deadline: 1733959254514, exception=java.io.IOException: Replication peer modification disabled 2024-12-11T23:19:54,525 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 8 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T23:19:54,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] ipc.CallRunner(138): callId: 314 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.2:41904 deadline: 1733959254638, exception=java.io.IOException: Replication peer modification disabled 2024-12-11T23:19:54,642 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 127 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T23:19:54,644 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-12-11T23:19:54.526Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T23:19:54.643Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T23:19:54,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-11T23:19:54,650 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:54,651 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=110, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:54,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=110, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-11T23:19:54,652 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-11T23:19:54,757 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-11T23:19:54,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-11T23:19:54,759 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:54,760 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=111, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:54,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=111, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-11T23:19:54,761 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-11T23:19:54,867 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:54,882 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=394 (was 394), OpenFileDescriptor=677 (was 677), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=145 (was 145), ProcessCount=11 (was 11), AvailableMemoryMB=8773 (was 8779) 2024-12-11T23:19:54,892 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=394, OpenFileDescriptor=677, MaxFileDescriptor=1048576, SystemLoadAverage=145, ProcessCount=11, AvailableMemoryMB=8773 2024-12-11T23:19:54,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-11T23:19:54,896 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:54,898 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x392daa89 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@25979a36 2024-12-11T23:19:54,944 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x392daa89 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x392daa89 to 127.0.0.1:50115 2024-12-11T23:19:54,944 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:55,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-11T23:19:55,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=113 2024-12-11T23:19:55,162 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-11T23:19:55,195 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:55,198 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=113 2024-12-11T23:19:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=113 2024-12-11T23:19:55,200 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:55,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-11T23:19:55,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 195 msec 2024-12-11T23:19:55,202 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-11T23:19:55,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 309 msec 2024-12-11T23:19:55,213 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:55,215 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x35103725 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@218f65f3 2024-12-11T23:19:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-11T23:19:55,217 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:55,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:55,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:55,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-11T23:19:55,233 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:55,233 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:55,233 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:55,233 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:55,234 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:55,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:55,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:55,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:55,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:55,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:55,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-11T23:19:55,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=115 2024-12-11T23:19:55,482 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-11T23:19:55,482 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-11T23:19:55,482 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Replication stream was removed by a user 2024-12-11T23:19:55,483 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:55,483 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:55,486 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x26549ef3 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f882aad 2024-12-11T23:19:55,487 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:55,488 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSourceManager(562): Done with the queue 1-d462fa545078,39401,1733959172007 2024-12-11T23:19:55,491 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=115 2024-12-11T23:19:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=115 2024-12-11T23:19:55,493 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:55,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-11T23:19:55,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 166 msec 2024-12-11T23:19:55,495 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-11T23:19:55,496 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-11T23:19:55,503 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-11T23:19:55,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 285 msec 2024-12-11T23:19:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-11T23:19:55,538 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:55,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-11T23:19:55,543 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:55,545 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=116, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:55,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=116, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-11T23:19:55,547 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-11T23:19:55,649 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:55,671 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=400 (was 394) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x26549ef3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x26549ef3-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x35103725-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x26549ef3-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x35103725-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x35103725 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=685 (was 677) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=150 (was 145) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8758 (was 8773) 2024-12-11T23:19:55,683 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=400, OpenFileDescriptor=685, MaxFileDescriptor=1048576, SystemLoadAverage=150, ProcessCount=11, AvailableMemoryMB=8757 2024-12-11T23:19:55,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-11T23:19:55,687 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:55,688 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x517fb003 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4fd31f17 2024-12-11T23:19:55,702 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x517fb003 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x517fb003 to 127.0.0.1:50115 2024-12-11T23:19:55,702 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:55,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-11T23:19:55,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=118 2024-12-11T23:19:55,892 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-11T23:19:55,923 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:55,926 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-11T23:19:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-11T23:19:55,928 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:55,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-11T23:19:55,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 192 msec 2024-12-11T23:19:55,929 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-11T23:19:55,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 245 msec 2024-12-11T23:19:55,938 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:55,939 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3eff4389 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6116fd00 2024-12-11T23:19:55,948 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:19:55,949 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:19:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-11T23:19:56,008 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:56,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:56,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-11T23:19:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:19:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-11T23:19:56,013 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:56,014 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5889c2fc to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@16697929 2024-12-11T23:19:56,028 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x5889c2fc {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5889c2fc to 127.0.0.1:50115 2024-12-11T23:19:56,028 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:19:56,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-11T23:19:56,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=120 2024-12-11T23:19:56,205 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:19:56,208 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-11T23:19:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-11T23:19:56,214 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:56,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-11T23:19:56,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 165 msec 2024-12-11T23:19:56,219 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-11T23:19:56,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 210 msec 2024-12-11T23:19:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-11T23:19:56,328 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:19:56,330 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:19:56,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T23:19:56,339 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:56,340 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:56,341 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:56,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:56,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:56,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:56,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:56,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:56,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T23:19:56,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=122 2024-12-11T23:19:56,510 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-11T23:19:56,510 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-11T23:19:56,510 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Replication stream was removed by a user 2024-12-11T23:19:56,511 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:19:56,511 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:19:56,515 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3432dc53 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1ab6a529 2024-12-11T23:19:56,515 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:19:56,516 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSourceManager(562): Done with the queue 1-d462fa545078,39401,1733959172007 2024-12-11T23:19:56,519 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-11T23:19:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-11T23:19:56,523 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:56,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-11T23:19:56,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 167 msec 2024-12-11T23:19:56,524 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-11T23:19:56,524 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-11T23:19:56,529 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-11T23:19:56,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 197 msec 2024-12-11T23:19:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T23:19:56,658 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:56,659 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T23:19:56,663 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:56,665 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=123, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:56,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=123, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-11T23:19:56,667 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T23:19:56,768 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-11T23:19:56,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:19:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-11T23:19:56,774 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:19:56,776 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=124, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:56,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=124, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-11T23:19:56,778 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-11T23:19:56,877 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:19:56,892 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=406 (was 400) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3432dc53 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3eff4389-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3eff4389 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3eff4389-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3432dc53-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50115@0x3432dc53-SendThread(127.0.0.1:50115) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=693 (was 685) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=150 (was 150), ProcessCount=11 (was 11), AvailableMemoryMB=8745 (was 8757) 2024-12-11T23:19:56,903 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=406, OpenFileDescriptor=693, MaxFileDescriptor=1048576, SystemLoadAverage=150, ProcessCount=11, AvailableMemoryMB=8742 2024-12-11T23:19:56,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T23:19:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T23:19:57,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T23:19:57,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=126 2024-12-11T23:19:57,316 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-11T23:19:57,352 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:19:57,354 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-11T23:19:57,354 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(613): peerId=1, queueId=1-d462fa545078,39401,1733959172007 (queues=1) is replicating from cluster=7c7d4594-c434-4a58-99e5-d3b675ab8ed8 to cluster=7c7d4594-c434-4a58-99e5-d3b675ab8ed8 2024-12-11T23:19:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-11T23:19:57,356 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(382): peerId=1, starting shipping worker for walGroupId=d462fa545078%2C39401%2C1733959172007 2024-12-11T23:19:57,356 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:57,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-11T23:19:57,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 196 msec 2024-12-11T23:19:57,358 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-11T23:19:57,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 454 msec 2024-12-11T23:19:57,360 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=1-d462fa545078,39401,1733959172007, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-12-11T23:19:57,362 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007.replicationSource.shipperd462fa545078%2C39401%2C1733959172007,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: d462fa545078%2C39401%2C1733959172007 2024-12-11T23:19:57,366 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007.replicationSource.wal-reader.d462fa545078%2C39401%2C1733959172007,1-d462fa545078,39401,1733959172007 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/WALs/d462fa545078,39401,1733959172007/d462fa545078%2C39401%2C1733959172007.1733959175249, startPosition=0, beingWritten=true 2024-12-11T23:19:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T23:19:57,538 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:19:57,540 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:19:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:19:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T23:19:57,546 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-11T23:19:57,547 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=127, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:19:57,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=127, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=8 msec 2024-12-11T23:19:57,549 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:19:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T23:19:57,657 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-12-11T23:19:57,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:19:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:19:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-11T23:19:57,663 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:57,664 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:57,664 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:57,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:19:57,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:57,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-11T23:19:57,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:19:57,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:19:57,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:19:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-11T23:19:57,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=129 2024-12-11T23:19:57,871 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-11T23:19:57,871 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-11T23:19:57,871 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Replication stream was removed by a user 2024-12-11T23:19:57,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-11T23:19:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-11T23:19:58,366 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T23:19:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-11T23:19:58,872 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007.replicationSource.wal-reader.d462fa545078%2C39401%2C1733959172007,1-d462fa545078,39401,1733959172007 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) ~[classes/:?] 2024-12-11T23:19:58,872 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007.replicationSource.shipperd462fa545078%2C39401%2C1733959172007,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:309) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-12-11T23:19:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:19:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-11T23:19:59,872 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSource(739): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007.replicationSource.shipperd462fa545078%2C39401%2C1733959172007,1-d462fa545078,39401,1733959172007 terminated 2024-12-11T23:19:59,872 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSourceManager(562): Done with the queue 1-d462fa545078,39401,1733959172007 2024-12-11T23:19:59,876 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=129 2024-12-11T23:19:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=129 2024-12-11T23:19:59,881 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:19:59,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-12-11T23:19:59,885 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-11T23:19:59,885 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-11T23:19:59,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1650 sec 2024-12-11T23:19:59,893 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-11T23:19:59,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.2350 sec 2024-12-11T23:20:00,388 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:20:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-11T23:20:01,836 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:20:01,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-11T23:20:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:20:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-11T23:20:01,839 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T23:20:01,840 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=130, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-11T23:20:01,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=130, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-11T23:20:01,842 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-11T23:20:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-11T23:20:01,946 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-11T23:20:01,962 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=410 (was 406) - Thread LEAK? -, OpenFileDescriptor=700 (was 693) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 150), ProcessCount=11 (was 11), AvailableMemoryMB=8691 (was 8742) 2024-12-11T23:20:01,973 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=410, OpenFileDescriptor=700, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=8689 2024-12-11T23:20:01,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-11T23:20:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-11T23:20:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-11T23:20:01,977 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:01,978 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x26814953 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@51def22a 2024-12-11T23:20:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-11T23:20:02,112 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x26814953 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x26814953 to 127.0.0.1:50115 2024-12-11T23:20:02,112 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:20:02,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:20:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-11T23:20:02,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=132 2024-12-11T23:20:02,297 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-11T23:20:02,323 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:20:02,325 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-11T23:20:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-11T23:20:02,328 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:20:02,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-11T23:20:02,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec 2024-12-11T23:20:02,330 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-11T23:20:02,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 355 msec 2024-12-11T23:20:02,339 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:02,341 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x11ab1754 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@224a4615 2024-12-11T23:20:02,354 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:20:02,354 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:20:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-11T23:20:02,607 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:20:02,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:20:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:20:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-11T23:20:02,610 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:02,611 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1affca9c to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5c391d89 2024-12-11T23:20:02,670 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x1affca9c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1affca9c to 127.0.0.1:50115 2024-12-11T23:20:02,670 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:20:02,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:20:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-11T23:20:02,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=134 2024-12-11T23:20:02,844 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:20:02,845 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:20:02,845 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:20:02,846 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:20:02,846 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:02,848 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x766cde9a to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@74d18767 2024-12-11T23:20:02,848 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:20:02,864 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:20:02,864 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:20:02,864 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-11T23:20:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-11T23:20:02,867 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:20:02,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-11T23:20:02,869 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:20:02,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 176 msec 2024-12-11T23:20:02,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 262 msec 2024-12-11T23:20:02,877 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:02,878 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x70084e07 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@111a1c35 2024-12-11T23:20:02,906 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:20:02,906 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:20:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-11T23:20:02,926 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:20:02,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:02,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:02,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:02,929 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-11T23:20:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:20:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-11T23:20:02,932 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:02,933 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x234a18ac to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@69842934 2024-12-11T23:20:02,944 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x234a18ac {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x234a18ac to 127.0.0.1:50115 2024-12-11T23:20:02,944 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:20:02,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:20:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-11T23:20:03,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=136 2024-12-11T23:20:03,119 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:20:03,120 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:20:03,120 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:20:03,120 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:20:03,120 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:03,121 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ddadebd to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@70ed8043 2024-12-11T23:20:03,122 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:20:03,137 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:20:03,137 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:20:03,137 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-11T23:20:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-11T23:20:03,140 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:20:03,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-11T23:20:03,142 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-11T23:20:03,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 175 msec 2024-12-11T23:20:03,143 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 213 msec 2024-12-11T23:20:03,151 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:03,152 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1607970c to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a01e380 2024-12-11T23:20:03,165 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:20:03,165 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:20:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-11T23:20:03,246 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:20:03,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:03,248 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:03,248 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:03,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:03,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-11T23:20:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:20:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-11T23:20:03,253 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:03,254 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0260a117 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e73ae89 2024-12-11T23:20:03,264 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x0260a117 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0260a117 to 127.0.0.1:50115 2024-12-11T23:20:03,264 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:20:03,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:20:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-11T23:20:03,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=138 2024-12-11T23:20:03,438 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:20:03,439 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:20:03,439 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:20:03,439 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:20:03,439 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:03,441 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x532806a7 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2d538d0e 2024-12-11T23:20:03,441 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:20:03,456 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:20:03,456 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:20:03,457 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-11T23:20:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-11T23:20:03,459 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:20:03,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-11T23:20:03,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 174 msec 2024-12-11T23:20:03,461 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-11T23:20:03,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 211 msec 2024-12-11T23:20:03,470 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:03,471 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x57cb481d to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2dc8337f 2024-12-11T23:20:03,485 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:20:03,485 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:20:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-11T23:20:03,567 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:20:03,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:03,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:03,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:03,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:20:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:20:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-11T23:20:03,573 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:03,574 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4f6995dc to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4b77db45 2024-12-11T23:20:03,586 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x4f6995dc {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4f6995dc to 127.0.0.1:50115 2024-12-11T23:20:03,586 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:20:03,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:20:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-11T23:20:03,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=140 2024-12-11T23:20:03,760 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:20:03,761 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:20:03,761 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:20:03,761 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:20:03,761 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:03,762 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x65ebe9a2 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1ccc50ee 2024-12-11T23:20:03,763 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:20:03,778 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:20:03,778 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:20:03,778 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-12-11T23:20:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-12-11T23:20:03,781 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:20:03,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-11T23:20:03,783 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:20:03,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 174 msec 2024-12-11T23:20:03,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 214 msec 2024-12-11T23:20:03,813 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:03,814 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x64d2e993 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@417627a5 2024-12-11T23:20:03,828 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:20:03,828 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:20:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-11T23:20:03,887 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:20:03,887 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:03,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:03,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-11T23:20:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:20:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-11T23:20:03,891 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:03,892 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x216b92dd to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@21f50851 2024-12-11T23:20:03,906 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x216b92dd {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x216b92dd to 127.0.0.1:50115 2024-12-11T23:20:03,906 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:20:03,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:20:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-11T23:20:04,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=142 2024-12-11T23:20:04,080 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:20:04,081 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:20:04,081 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:20:04,081 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:20:04,081 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:04,082 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x34dda0cb to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7c7b17a5 2024-12-11T23:20:04,082 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:20:04,096 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:20:04,097 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:20:04,097 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-12-11T23:20:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-12-11T23:20:04,099 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:20:04,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-11T23:20:04,102 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-11T23:20:04,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 172 msec 2024-12-11T23:20:04,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 213 msec 2024-12-11T23:20:04,110 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:04,111 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6a54443c to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5d51c790 2024-12-11T23:20:04,123 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:20:04,123 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:20:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-11T23:20:04,206 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:20:04,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:04,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:20:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-11T23:20:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-11T23:20:04,210 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:04,211 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x56a62768 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b7f9133 2024-12-11T23:20:04,222 DEBUG [ReadOnlyZKClient-127.0.0.1:50115@0x56a62768 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x56a62768 to 127.0.0.1:50115 2024-12-11T23:20:04,222 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T23:20:04,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:20:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-11T23:20:04,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=144 2024-12-11T23:20:04,395 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-11T23:20:04,396 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-11T23:20:04,397 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-11T23:20:04,397 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:20:04,397 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:04,398 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f8d7d94 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1f65975d 2024-12-11T23:20:04,398 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:20:04,413 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(231): queueId=1-d462fa545078,39401,1733959172007, ReplicationSource: 1, currentBandwidth=0 2024-12-11T23:20:04,414 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-11T23:20:04,414 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-12-11T23:20:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-12-11T23:20:04,416 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:20:04,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-11T23:20:04,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 173 msec 2024-12-11T23:20:04,418 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:50115/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-11T23:20:04,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 211 msec 2024-12-11T23:20:04,427 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:04,428 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x48659abb to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@10538d3e 2024-12-11T23:20:04,438 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 1 more 2024-12-11T23:20:04,438 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-11T23:20:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-11T23:20:04,527 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-11T23:20:04,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-11T23:20:04,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:20:04,528 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-11T23:20:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:20:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:20:04,529 DEBUG [PEWorker-4 {}] procedure.MasterProcedureScheduler(648): PEER '1', shared lock count=0, exclusively locked by procId=145 2024-12-11T23:20:04,529 DEBUG [PEWorker-4 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=146, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-11T23:20:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-11T23:20:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:20:04,532 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:20:04,532 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-11T23:20:04,532 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:20:04,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-11T23:20:04,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:20:04,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-11T23:20:04,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-11T23:20:04,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41885-0x100172d404c0000, quorum=127.0.0.1:50115, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-11T23:20:04,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-11T23:20:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-11T23:20:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:20:04,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=147 2024-12-11T23:20:04,696 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-11T23:20:04,696 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-11T23:20:04,696 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-d462fa545078,39401,1733959172007 because: Replication stream was removed by a user 2024-12-11T23:20:04,697 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-11T23:20:04,697 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:50115' and parent='/hbase-test1' 2024-12-11T23:20:04,698 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3939f0b6 to 127.0.0.1:50115 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@631a6315 2024-12-11T23:20:04,699 WARN [RS_REFRESH_PEER-regionserver/d462fa545078:0-0.replicationSource,1-d462fa545078,39401,1733959172007 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 7 more 2024-12-11T23:20:04,699 INFO [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSourceManager(562): Done with the queue 1-d462fa545078,39401,1733959172007 2024-12-11T23:20:04,701 DEBUG [RS_REFRESH_PEER-regionserver/d462fa545078:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-11T23:20:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-11T23:20:04,703 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on d462fa545078,39401,1733959172007 suceeded 2024-12-11T23:20:04,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-12-11T23:20:04,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec 2024-12-11T23:20:04,705 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-11T23:20:04,705 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-11T23:20:04,711 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-11T23:20:04,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 183 msec 2024-12-11T23:20:04,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-11T23:20:04,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:20:04,847 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-11T23:20:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:20:05,374 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T23:20:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:20:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:20:08,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:20:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:20:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:20:16,545 INFO [master/d462fa545078:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-11T23:20:16,545 INFO [master/d462fa545078:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-11T23:20:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:20:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:20:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:20:30,388 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:20:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:20:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:20:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:20:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:21:00,389 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:21:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:21:13,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:21:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:21:30,389 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:21:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:21:35,098 WARN [regionserver/d462fa545078:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-11T23:21:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:21:53,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:22:00,389 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:22:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:22:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:22:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:22:30,390 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:22:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:22:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:22:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:23:00,390 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:23:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:23:13,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:23:30,391 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:23:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:23:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:23:35,123 WARN [regionserver/d462fa545078:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 1, running: 0 2024-12-11T23:23:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:23:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:24:00,391 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:24:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:24:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:24:30,391 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:24:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:24:32,094 INFO [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=660.99 KB, freeSize=879.35 MB, max=880 MB, blockCount=0, accesses=0, hits=0, hitRatio=0, cachingAccesses=0, cachingHits=0, cachingHitsRatio=0,evictions=29, evicted=0, evictedPerRun=0.0 2024-12-11T23:24:32,177 DEBUG [master/d462fa545078:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=0, created chunk count=9, reused chunk count=0, reuseRatio=0.00% 2024-12-11T23:24:32,177 DEBUG [master/d462fa545078:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-12-11T23:24:34,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:24:35,115 INFO [regionserver/d462fa545078:0.Chore.1 {}] regionserver.Replication$ReplicationStatisticsChore(208): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B 2024-12-11T23:24:36,783 DEBUG [master/d462fa545078:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-11T23:24:36,783 INFO [master/d462fa545078:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-11T23:24:36,784 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-11T23:24:36,787 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(204): Hosts are {d462fa545078=0} racks are {/default-rack=0} 2024-12-11T23:24:36,792 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(303): server 0 has 2 regions 2024-12-11T23:24:36,792 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T23:24:36,792 INFO [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T23:24:36,792 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-11T23:24:36,794 INFO [master/d462fa545078:0.Chore.1 {}] balancer.StochasticLoadBalancer(357): Not running balancer because only 1 active regionserver(s) 2024-12-11T23:24:36,795 DEBUG [master/d462fa545078:0.Chore.1 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-11T23:24:36,796 DEBUG [master/d462fa545078:0.Chore.1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T23:24:36,797 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45801, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T23:24:36,814 DEBUG [master/d462fa545078:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T23:24:37,173 INFO [regionserver/d462fa545078:0.Chore.1 {}] regionserver.HRegionServer$PeriodicMemStoreFlusher(1763): MemstoreFlusherChore requesting flush of hbase:meta,,1.1588230740 because 1588230740/ns has an old edit so flush to free WALs after random delay 102465 ms 2024-12-11T23:24:39,123 INFO [regionserver/d462fa545078:0.Chore.1 {}] regionserver.HRegionServer$PeriodicMemStoreFlusher(1763): MemstoreFlusherChore requesting flush of hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05. because 301a3ed59652d1a1ff0f40a1ccd90f05/queue has an old edit so flush to free WALs after random delay 117973 ms 2024-12-11T23:24:41,817 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T23:24:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:24:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:25:00,392 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:25:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:25:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:25:30,392 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:25:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:25:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:25:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:25:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:26:00,393 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:26:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:26:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:26:19,640 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-11T23:26:19,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.34 KB heapSize=3.38 KB 2024-12-11T23:26:19,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/.tmp/info/59a55679173842d1b6046a691386256b is 147, key is hbase:replication,,1733959176800.301a3ed59652d1a1ff0f40a1ccd90f05./info:regioninfo/1733959178054/Put/seqid=0 2024-12-11T23:26:19,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741838_1014 (size=6631) 2024-12-11T23:26:20,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.17 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/.tmp/info/59a55679173842d1b6046a691386256b 2024-12-11T23:26:20,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/.tmp/ns/244195b99ee74a4f86b7aba06a8de6fc is 43, key is default/ns:d/1733959176445/Put/seqid=0 2024-12-11T23:26:20,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741839_1015 (size=5153) 2024-12-11T23:26:20,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/.tmp/ns/244195b99ee74a4f86b7aba06a8de6fc 2024-12-11T23:26:20,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/.tmp/table/73a76ae3500a483383bb27e2e2648daa is 53, key is hbase:replication/table:state/1733959178073/Put/seqid=0 2024-12-11T23:26:20,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741840_1016 (size=5256) 2024-12-11T23:26:21,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=98 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/.tmp/table/73a76ae3500a483383bb27e2e2648daa 2024-12-11T23:26:21,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/.tmp/info/59a55679173842d1b6046a691386256b as hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/info/59a55679173842d1b6046a691386256b 2024-12-11T23:26:21,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/info/59a55679173842d1b6046a691386256b, entries=10, sequenceid=11, filesize=6.5 K 2024-12-11T23:26:21,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/.tmp/ns/244195b99ee74a4f86b7aba06a8de6fc as hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/ns/244195b99ee74a4f86b7aba06a8de6fc 2024-12-11T23:26:21,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/ns/244195b99ee74a4f86b7aba06a8de6fc, entries=2, sequenceid=11, filesize=5.0 K 2024-12-11T23:26:21,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/.tmp/table/73a76ae3500a483383bb27e2e2648daa as hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/table/73a76ae3500a483383bb27e2e2648daa 2024-12-11T23:26:21,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/meta/1588230740/table/73a76ae3500a483383bb27e2e2648daa, entries=2, sequenceid=11, filesize=5.1 K 2024-12-11T23:26:21,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.34 KB/1368, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 1432ms, sequenceid=11, compaction requested=false 2024-12-11T23:26:21,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-12-11T23:26:21,289 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:26:30,393 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:26:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:26:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:26:37,097 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-11T23:26:37,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 301a3ed59652d1a1ff0f40a1ccd90f05 3/3 column families, dataSize=2.75 KB heapSize=6.19 KB 2024-12-11T23:26:37,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/replication/301a3ed59652d1a1ff0f40a1ccd90f05/.tmp/hfileref/5a0ad02d8fbb43e89e60e29752f69ef4 is 33, key is 1/hfileref:/1733959204710/DeleteFamily/seqid=0 2024-12-11T23:26:37,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741841_1017 (size=5433) 2024-12-11T23:26:37,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=348 B at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/replication/301a3ed59652d1a1ff0f40a1ccd90f05/.tmp/hfileref/5a0ad02d8fbb43e89e60e29752f69ef4 2024-12-11T23:26:37,545 INFO [MemStoreFlusher.0 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5a0ad02d8fbb43e89e60e29752f69ef4 2024-12-11T23:26:37,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/replication/301a3ed59652d1a1ff0f40a1ccd90f05/.tmp/queue/2d61b4cb13754843aabc997119c84bd2 is 63, key is 1-d462fa545078,39401,1733959172007/queue:/1733959204700/DeleteFamily/seqid=0 2024-12-11T23:26:37,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741842_1018 (size=5982) 2024-12-11T23:26:37,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.41 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/replication/301a3ed59652d1a1ff0f40a1ccd90f05/.tmp/queue/2d61b4cb13754843aabc997119c84bd2 2024-12-11T23:26:37,981 INFO [MemStoreFlusher.0 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2d61b4cb13754843aabc997119c84bd2 2024-12-11T23:26:37,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/replication/301a3ed59652d1a1ff0f40a1ccd90f05/.tmp/hfileref/5a0ad02d8fbb43e89e60e29752f69ef4 as hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/replication/301a3ed59652d1a1ff0f40a1ccd90f05/hfileref/5a0ad02d8fbb43e89e60e29752f69ef4 2024-12-11T23:26:37,991 INFO [MemStoreFlusher.0 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5a0ad02d8fbb43e89e60e29752f69ef4 2024-12-11T23:26:37,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/replication/301a3ed59652d1a1ff0f40a1ccd90f05/hfileref/5a0ad02d8fbb43e89e60e29752f69ef4, entries=12, sequenceid=40, filesize=5.3 K 2024-12-11T23:26:37,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/replication/301a3ed59652d1a1ff0f40a1ccd90f05/.tmp/queue/2d61b4cb13754843aabc997119c84bd2 as hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/replication/301a3ed59652d1a1ff0f40a1ccd90f05/queue/2d61b4cb13754843aabc997119c84bd2 2024-12-11T23:26:38,001 INFO [MemStoreFlusher.0 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2d61b4cb13754843aabc997119c84bd2 2024-12-11T23:26:38,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/data/hbase/replication/301a3ed59652d1a1ff0f40a1ccd90f05/queue/2d61b4cb13754843aabc997119c84bd2, entries=12, sequenceid=40, filesize=5.8 K 2024-12-11T23:26:38,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.75 KB/2820, heapSize ~5.91 KB/6048, currentSize=0 B/0 for 301a3ed59652d1a1ff0f40a1ccd90f05 in 905ms, sequenceid=40, compaction requested=false 2024-12-11T23:26:38,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 301a3ed59652d1a1ff0f40a1ccd90f05: 2024-12-11T23:26:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:26:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:27:00,393 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:27:06,290 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:27:08,025 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:27:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:27:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:27:30,394 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:27:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:27:35,183 WARN [regionserver/d462fa545078:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-12-11T23:27:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:27:51,290 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:27:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:27:53,026 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:27:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:28:00,394 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:28:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:28:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:28:30,395 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:28:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:28:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:28:36,291 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:28:38,026 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:28:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:28:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:29:00,395 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:29:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:29:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:29:21,291 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:29:23,026 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:29:30,395 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:29:32,081 INFO [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=660.99 KB, freeSize=879.35 MB, max=880 MB, blockCount=0, accesses=0, hits=0, hitRatio=0, cachingAccesses=0, cachingHits=0, cachingHitsRatio=0,evictions=59, evicted=0, evictedPerRun=0.0 2024-12-11T23:29:32,177 DEBUG [master/d462fa545078:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=5, created chunk count=9, reused chunk count=0, reuseRatio=0.00% 2024-12-11T23:29:32,177 DEBUG [master/d462fa545078:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-12-11T23:29:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:29:34,993 DEBUG [master/d462fa545078:0.Chore.1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T23:29:34,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60339, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T23:29:35,112 WARN [regionserver/d462fa545078:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-11T23:29:35,114 INFO [regionserver/d462fa545078:0.Chore.1 {}] regionserver.Replication$ReplicationStatisticsChore(208): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B 2024-12-11T23:29:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:29:36,588 DEBUG [master/d462fa545078:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-11T23:29:36,588 INFO [master/d462fa545078:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-11T23:29:36,588 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-11T23:29:36,589 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(204): Hosts are {d462fa545078=0} racks are {/default-rack=0} 2024-12-11T23:29:36,589 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(303): server 0 has 2 regions 2024-12-11T23:29:36,589 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T23:29:36,589 INFO [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T23:29:36,589 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-11T23:29:36,589 INFO [master/d462fa545078:0.Chore.1 {}] balancer.StochasticLoadBalancer(357): Not running balancer because only 1 active regionserver(s) 2024-12-11T23:29:36,589 DEBUG [master/d462fa545078:0.Chore.1 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-11T23:29:36,596 DEBUG [master/d462fa545078:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T23:29:41,800 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T23:29:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:29:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:30:00,396 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:30:06,292 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:30:08,027 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:30:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:30:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:30:30,396 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:30:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:30:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:30:51,292 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:30:53,028 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:30:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:30:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:31:00,396 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:31:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:31:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:31:30,397 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:31:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:31:35,101 WARN [regionserver/d462fa545078:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-11T23:31:36,293 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:31:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:31:38,028 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:31:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:31:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:32:00,397 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:32:13,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:32:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:32:21,293 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:32:23,029 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:32:30,398 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:32:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:32:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:32:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:32:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:33:00,398 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:33:06,294 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:33:08,029 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:33:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:33:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:33:30,398 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:33:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:33:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:33:51,295 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:33:53,030 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:33:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:33:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:34:00,399 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:34:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:34:17,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:34:30,399 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:34:32,081 INFO [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=665.45 KB, freeSize=879.35 MB, max=880 MB, blockCount=4, accesses=5, hits=1, hitRatio=20.00%, , cachingAccesses=5, cachingHits=1, cachingHitsRatio=20.00%, evictions=89, evicted=0, evictedPerRun=0.0 2024-12-11T23:34:32,177 DEBUG [master/d462fa545078:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=5, created chunk count=9, reused chunk count=0, reuseRatio=0.00% 2024-12-11T23:34:32,177 DEBUG [master/d462fa545078:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-12-11T23:34:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:34:34,235 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(201): WAL AsyncFSWAL d462fa545078%2C41885%2C1733959171253:(num 1733959174134) roll period 900000 ms elapsed 2024-12-11T23:34:34,256 DEBUG [master:store-WAL-Roller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/WALs/d462fa545078,41885,1733959171253/d462fa545078%2C41885%2C1733959171253.1733960074236, exclude list is [], retry=0 2024-12-11T23:34:34,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44285,DS-ec823315-b9ca-45af-9c64-68e2274b0859,DISK] 2024-12-11T23:34:34,261 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/WALs/d462fa545078,41885,1733959171253/d462fa545078%2C41885%2C1733959171253.1733959174134 with entries=1144, filesize=507.65 KB; new WAL /user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/WALs/d462fa545078,41885,1733959171253/d462fa545078%2C41885%2C1733959171253.1733960074236 2024-12-11T23:34:34,262 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43107:43107)] 2024-12-11T23:34:34,262 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/WALs/d462fa545078,41885,1733959171253/d462fa545078%2C41885%2C1733959171253.1733959174134 is not closed yet, will try archiving it next time 2024-12-11T23:34:34,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741830_1006 (size=519846) 2024-12-11T23:34:34,311 INFO [master:store-Flusher {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=434.28 KB heapSize=534.32 KB 2024-12-11T23:34:34,332 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c02ece193a94a7480033777c5ed5245 is 82, key is hbase:meta,,1/info:regioninfo/1733959176333/Put/seqid=0 2024-12-11T23:34:34,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741844_1020 (size=5672) 2024-12-11T23:34:34,739 INFO [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1147 (bloomFilter=true), to=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c02ece193a94a7480033777c5ed5245 2024-12-11T23:34:34,773 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/56be616592da4ea89757581af65ac701 is 293, key is \x00\x00\x00\x00\x00\x00\x00\x92/proc:d/1733959204529/Put/seqid=0 2024-12-11T23:34:34,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741845_1021 (size=11470) 2024-12-11T23:34:35,114 INFO [regionserver/d462fa545078:0.Chore.1 {}] regionserver.Replication$ReplicationStatisticsChore(208): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B 2024-12-11T23:34:35,190 INFO [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=432.73 KB at sequenceid=1147 (bloomFilter=true), to=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/56be616592da4ea89757581af65ac701 2024-12-11T23:34:35,199 INFO [master:store-Flusher {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 56be616592da4ea89757581af65ac701 2024-12-11T23:34:35,217 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e6fe421b37d8409cb1333dcb5942fc0d is 69, key is d462fa545078,39401,1733959172007/rs:state/1733959175026/Put/seqid=0 2024-12-11T23:34:35,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741846_1022 (size=5156) 2024-12-11T23:34:35,623 INFO [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1147 (bloomFilter=true), to=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e6fe421b37d8409cb1333dcb5942fc0d 2024-12-11T23:34:35,651 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a31c0befeb484d4ab3d12d22aa10c19a is 68, key is replication_peer_modification_on/state:d/1733959201949/Put/seqid=0 2024-12-11T23:34:35,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44285 is added to blk_1073741847_1023 (size=5154) 2024-12-11T23:34:36,059 INFO [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1 KB at sequenceid=1147 (bloomFilter=true), to=hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a31c0befeb484d4ab3d12d22aa10c19a 2024-12-11T23:34:36,074 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c02ece193a94a7480033777c5ed5245 as hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c02ece193a94a7480033777c5ed5245 2024-12-11T23:34:36,082 INFO [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c02ece193a94a7480033777c5ed5245, entries=8, sequenceid=1147, filesize=5.5 K 2024-12-11T23:34:36,084 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/56be616592da4ea89757581af65ac701 as hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/56be616592da4ea89757581af65ac701 2024-12-11T23:34:36,091 INFO [master:store-Flusher {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 56be616592da4ea89757581af65ac701 2024-12-11T23:34:36,091 INFO [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/56be616592da4ea89757581af65ac701, entries=147, sequenceid=1147, filesize=11.2 K 2024-12-11T23:34:36,093 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e6fe421b37d8409cb1333dcb5942fc0d as hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e6fe421b37d8409cb1333dcb5942fc0d 2024-12-11T23:34:36,099 INFO [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e6fe421b37d8409cb1333dcb5942fc0d, entries=1, sequenceid=1147, filesize=5.0 K 2024-12-11T23:34:36,101 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a31c0befeb484d4ab3d12d22aa10c19a as hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a31c0befeb484d4ab3d12d22aa10c19a 2024-12-11T23:34:36,107 INFO [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a31c0befeb484d4ab3d12d22aa10c19a, entries=1, sequenceid=1147, filesize=5.0 K 2024-12-11T23:34:36,108 INFO [master:store-Flusher {}] regionserver.HRegion(3140): Finished flush of dataSize ~434.28 KB/444706, heapSize ~534.26 KB/547080, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1797ms, sequenceid=1147, compaction requested=false 2024-12-11T23:34:36,108 DEBUG [master:store-Flusher {}] regionserver.HRegion(2603): Flush status journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-11T23:34:36,295 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:34:36,962 DEBUG [master/d462fa545078:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-11T23:34:36,962 INFO [master/d462fa545078:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-11T23:34:36,962 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-11T23:34:36,963 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(204): Hosts are {d462fa545078=0} racks are {/default-rack=0} 2024-12-11T23:34:36,963 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(303): server 0 has 2 regions 2024-12-11T23:34:36,963 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T23:34:36,963 INFO [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T23:34:36,963 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-11T23:34:36,964 INFO [master/d462fa545078:0.Chore.1 {}] balancer.StochasticLoadBalancer(357): Not running balancer because only 1 active regionserver(s) 2024-12-11T23:34:36,964 DEBUG [master/d462fa545078:0.Chore.1 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-11T23:34:36,967 DEBUG [master/d462fa545078:0.Chore.1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T23:34:36,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43243, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T23:34:36,973 DEBUG [master/d462fa545078:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T23:34:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:34:38,030 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:34:41,809 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T23:34:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:34:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:35:00,399 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:35:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:35:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:35:21,295 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:35:23,031 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:35:30,400 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:35:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:35:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:35:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:35:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:36:00,400 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:36:06,296 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:36:08,032 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:36:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:36:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:36:30,401 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:36:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:36:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:36:51,297 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:36:53,032 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:36:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:36:58,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:37:00,401 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:37:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:37:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:37:30,401 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:37:35,115 WARN [regionserver/d462fa545078:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 4, running: 1 2024-12-11T23:37:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:37:36,297 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:37:38,033 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:37:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:37:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:37:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:38:00,402 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:38:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:38:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:38:21,298 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:38:23,033 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:38:30,402 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:38:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:38:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:38:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:38:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:39:00,403 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:39:06,298 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:39:08,034 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:39:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:39:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:39:30,403 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:39:32,081 INFO [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=665.45 KB, freeSize=879.35 MB, max=880 MB, blockCount=4, accesses=7, hits=3, hitRatio=42.86%, , cachingAccesses=7, cachingHits=3, cachingHitsRatio=42.86%, evictions=119, evicted=0, evictedPerRun=0.0 2024-12-11T23:39:32,177 DEBUG [master/d462fa545078:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=8, created chunk count=9, reused chunk count=1, reuseRatio=10.00% 2024-12-11T23:39:32,177 DEBUG [master/d462fa545078:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-12-11T23:39:35,161 WARN [regionserver/d462fa545078:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 1, running: 0 2024-12-11T23:39:35,162 INFO [regionserver/d462fa545078:0.Chore.1 {}] regionserver.Replication$ReplicationStatisticsChore(208): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B 2024-12-11T23:39:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:39:36,517 DEBUG [master/d462fa545078:0.Chore.1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T23:39:36,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51251, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T23:39:36,524 DEBUG [master/d462fa545078:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T23:39:36,544 DEBUG [master/d462fa545078:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-11T23:39:36,544 INFO [master/d462fa545078:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-11T23:39:36,544 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-11T23:39:36,544 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(204): Hosts are {d462fa545078=0} racks are {/default-rack=0} 2024-12-11T23:39:36,544 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(303): server 0 has 2 regions 2024-12-11T23:39:36,544 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T23:39:36,544 INFO [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T23:39:36,544 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-11T23:39:36,544 INFO [master/d462fa545078:0.Chore.1 {}] balancer.StochasticLoadBalancer(357): Not running balancer because only 1 active regionserver(s) 2024-12-11T23:39:36,545 DEBUG [master/d462fa545078:0.Chore.1 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-11T23:39:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:39:41,808 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T23:39:51,299 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:39:53,034 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:39:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:39:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:40:00,404 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:40:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:40:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:40:30,404 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:40:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:40:36,299 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:40:38,035 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:40:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:40:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:40:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:41:00,405 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:41:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:41:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:41:21,300 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:41:23,035 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:41:30,405 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:41:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:41:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:41:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:41:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:42:00,406 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:42:06,300 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:42:08,036 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:42:16,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:42:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:42:30,406 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:42:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:42:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:42:51,301 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:42:53,036 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:42:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:43:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:43:00,406 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:43:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:43:20,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:43:30,407 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:43:36,301 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:43:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:43:38,037 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:43:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:43:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:44:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:44:00,407 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:44:17,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:44:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:44:21,302 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:44:23,038 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:44:30,407 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:44:32,081 INFO [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=665.45 KB, freeSize=879.35 MB, max=880 MB, blockCount=4, accesses=12, hits=8, hitRatio=66.67%, , cachingAccesses=12, cachingHits=8, cachingHitsRatio=66.67%, evictions=149, evicted=0, evictedPerRun=0.0 2024-12-11T23:44:32,177 DEBUG [master/d462fa545078:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=8, created chunk count=9, reused chunk count=1, reuseRatio=10.00% 2024-12-11T23:44:32,177 DEBUG [master/d462fa545078:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-12-11T23:44:35,115 INFO [regionserver/d462fa545078:0.Chore.1 {}] regionserver.Replication$ReplicationStatisticsChore(208): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B 2024-12-11T23:44:36,619 DEBUG [master/d462fa545078:0.Chore.1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T23:44:36,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44311, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T23:44:36,626 DEBUG [master/d462fa545078:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T23:44:36,640 DEBUG [master/d462fa545078:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-11T23:44:36,640 INFO [master/d462fa545078:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-11T23:44:36,640 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-11T23:44:36,640 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(204): Hosts are {d462fa545078=0} racks are {/default-rack=0} 2024-12-11T23:44:36,640 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(303): server 0 has 2 regions 2024-12-11T23:44:36,640 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T23:44:36,640 INFO [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T23:44:36,640 DEBUG [master/d462fa545078:0.Chore.1 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-11T23:44:36,641 INFO [master/d462fa545078:0.Chore.1 {}] balancer.StochasticLoadBalancer(357): Not running balancer because only 1 active regionserver(s) 2024-12-11T23:44:36,641 DEBUG [master/d462fa545078:0.Chore.1 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-11T23:44:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:44:40,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:44:41,817 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T23:44:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:45:00,408 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T23:45:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T23:45:06,303 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 17040 2024-12-11T23:45:08,038 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 301a3ed59652d1a1ff0f40a1ccd90f05, had cached 0 bytes from a total of 11415 2024-12-11T23:45:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T23:45:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 ====> TEST TIMED OUT. PRINTING THREAD DUMP. <==== Timestamp: 2024-12-11 11:45:27,155 "IPC Server handler 2 on default port 36649" daemon prio=5 tid=66 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "NIOWorkerThread-8" daemon prio=5 tid=164 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Block report processor" daemon prio=5 tid=51 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.processQueue(BlockManager.java:5627) at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.run(BlockManager.java:5614) "ReadOnlyZKClient-127.0.0.1:50115@0x0fe865dc" daemon prio=5 tid=413 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "surefire-forkedjvm-stream-flusher" daemon prio=5 tid=16 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "HMaster-EventLoopGroup-1-1" daemon prio=10 tid=154 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "M:0;d462fa545078:41885" daemon prio=5 tid=184 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:81) at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:64) at app//org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:625) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "process reaper" daemon prio=10 tid=1004 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) at java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@12a8d9f4" daemon prio=5 tid=72 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:4550) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x34de41ae" daemon prio=5 tid=385 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp971800375-89" daemon prio=5 tid=89 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PEWorker-4" daemon prio=5 tid=250 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) "NIOWorkerThread-14" daemon prio=5 tid=170 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x5baf0f43" daemon prio=5 tid=573 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x65ebe9a2" daemon prio=5 tid=708 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server idle connection scanner for port 35819" daemon prio=5 tid=96 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "ReadOnlyZKClient-127.0.0.1:50115@0x6ca9f519" daemon prio=5 tid=381 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x339aa5aa" daemon prio=5 tid=493 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ConnnectionExpirer" daemon prio=5 tid=137 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$ConnectionExpirerThread.run(NIOServerCnxnFactory.java:554) "PEWorker-3" daemon prio=5 tid=249 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) "pool-6-thread-1" prio=5 tid=46 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 3 on default port 35819" daemon prio=5 tid=107 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "ProcedureDispatcherTimeoutThread" daemon prio=5 tid=252 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) at app//org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher$TimeoutExecutorThread.run(RemoteProcedureDispatcher.java:328) "ReadOnlyZKClient-127.0.0.1:50115@0x6b10a65d" daemon prio=5 tid=620 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x26549ef3" daemon prio=5 tid=641 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1228916748-37" daemon prio=5 tid=37 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007fd69842a550.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp971800375-87" daemon prio=5 tid=87 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007fd69842a550.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/cluster_e4efebb6-834a-ad19-6c5c-52938933ef0f/data/data1/current/BP-1905828385-172.17.0.2-1733959167787" daemon prio=5 tid=123 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x735a914b" daemon prio=5 tid=375 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "DataXceiver for client DFSClient_NONMAPREDUCE_-793969893_22 at /127.0.0.1:39346 [Receiving block BP-1905828385-172.17.0.2-1733959167787:blk_1073741843_1019]" daemon prio=5 tid=889 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Notification Thread" daemon prio=9 tid=13 runnable java.lang.Thread.State: RUNNABLE "NIOWorkerThread-15" daemon prio=5 tid=171 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41885" daemon prio=5 tid=178 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39401" daemon prio=5 tid=202 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "RPCClient-NioEventLoopGroup-4-3" daemon prio=5 tid=306 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server listener on 0" daemon prio=5 tid=54 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559) "RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=39401" daemon prio=5 tid=205 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "regionserver/d462fa545078:0.leaseChecker" daemon prio=5 tid=276 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) "RS_REFRESH_PEER-regionserver/d462fa545078:0-0" daemon prio=5 tid=333 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39401" daemon prio=5 tid=198 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "AsyncFSWAL-0-hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187-prefix:d462fa545078,39401,1733959172007.rep" daemon prio=5 tid=328 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test-SendThread(127.0.0.1:50115)" daemon prio=5 tid=155 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) "ReadOnlyZKClient-127.0.0.1:50115@0x57cb481d" daemon prio=5 tid=702 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "MemStoreFlusher.0" daemon prio=5 tid=279 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:77) at app//org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:323) "pool-14-thread-1" daemon prio=5 tid=132 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp971800375-88-acceptor-0@6082fe79-ServerConnector@41fba10{HTTP/1.1, (http/1.1)}{localhost:35351}" daemon prio=3 tid=88 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "snapshot-hfile-cleaner-cache-refresher" daemon prio=5 tid=264 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "ReadOnlyZKClient-127.0.0.1:50115@0x57a18bac" daemon prio=5 tid=476 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "HMaster-EventLoopGroup-1-3" daemon prio=10 tid=304 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1228916748-39" daemon prio=5 tid=39 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007fd69842a550.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Socket Reader #1 for port 0" daemon prio=5 tid=95 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497) at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476) "d462fa545078:41885" daemon prio=5 tid=253 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager.waitOnAssignQueue(AssignmentManager.java:2398) at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager.processAssignQueue(AssignmentManager.java:2420) at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager$1.run(AssignmentManager.java:2360) "RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=41885" daemon prio=5 tid=182 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "ReadOnlyZKClient-127.0.0.1:50115@0x1607970c" daemon prio=5 tid=692 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "zk-event-processor-pool-0" daemon prio=5 tid=160 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "SessionTracker" daemon prio=5 tid=141 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) "RS_OPEN_PRIORITY_REGION-regionserver/d462fa545078:0-0" daemon prio=5 tid=325 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x2543c855" daemon prio=5 tid=365 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x05497410" daemon prio=5 tid=466 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1228916748-44" daemon prio=5 tid=44 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x03e0174b" daemon prio=5 tid=427 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x1b278bf3" daemon prio=5 tid=628 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x4810bfb1" daemon prio=5 tid=371 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x34dda0cb" daemon prio=5 tid=718 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39401" daemon prio=5 tid=201 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "ProcessThread(sid:0 cport:50115):" daemon prio=5 tid=143 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142) "NIOWorkerThread-6" daemon prio=5 tid=162 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x323c9517" daemon prio=5 tid=469 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 3 on default port 36649" daemon prio=5 tid=67 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "IPC Parameter Sending Thread for localhost/127.0.0.1:36649" daemon prio=5 tid=110 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) at java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) at app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Registry-endpoints-refresh-end-points" daemon prio=5 tid=302 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.client.RegistryEndpointsRefresher.mainLoop(RegistryEndpointsRefresher.java:92) at app//org.apache.hadoop.hbase.client.RegistryEndpointsRefresher$$Lambda$806/0x00007fd698bb6410.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x35103725" daemon prio=5 tid=638 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "regionserver/d462fa545078:0.procedureResultReporter" daemon prio=5 tid=278 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) "region-location-0" daemon prio=5 tid=301 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x70084e07" daemon prio=5 tid=682 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@3b6e4935" daemon prio=5 tid=74 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:4689) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "weak-ref-cleaner-strictcontextstorage" daemon prio=1 tid=152 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) at app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "SyncThread:0" daemon prio=5 tid=142 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170) "DataXceiver for client DFSClient_NONMAPREDUCE_365039782_22 at /127.0.0.1:40836 [Receiving block BP-1905828385-172.17.0.2-1733959167787:blk_1073741834_1010]" daemon prio=5 tid=296 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41885" daemon prio=5 tid=176 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "pool-17-thread-1" prio=5 tid=86 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "WorkerMonitor" daemon prio=5 tid=246 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:279) at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) at app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56) "RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41885" daemon prio=5 tid=174 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "pool-1-thread-1" daemon prio=5 tid=14 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.run(EPollPort.java:275) at java.base@17.0.11/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:113) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "CacheReplicationMonitor(761209294)" daemon prio=5 tid=75 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759) at app//org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:186) "PEWorker-5" daemon prio=5 tid=251 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) "ReadOnlyZKClient-127.0.0.1:50115@0x3432dc53" daemon prio=5 tid=654 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x3d0c31ff" daemon prio=5 tid=517 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x3379baa6" daemon prio=5 tid=391 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Session-HouseKeeper-5cc7bd5c-1" prio=5 tid=91 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x182271ee" daemon prio=5 tid=537 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-7" daemon prio=5 tid=163 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x3303870e" daemon prio=5 tid=527 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 0 on default port 36649" daemon prio=5 tid=64 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41885" daemon prio=5 tid=175 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "master/d462fa545078:0:becomeActiveMaster-MemStoreChunkPool Statistics" daemon prio=5 tid=214 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "main" prio=5 tid=1 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/java.lang.Thread.dumpThreads(Native Method) at java.base@17.0.11/java.lang.Thread.getAllStackTraces(Thread.java:1671) at app//org.apache.hadoop.hbase.TimedOutTestsListener.buildThreadDump(TimedOutTestsListener.java:92) at app//org.apache.hadoop.hbase.TimedOutTestsListener.buildThreadDiagnosticString(TimedOutTestsListener.java:78) at app//org.apache.hadoop.hbase.TimedOutTestsListener.testFailure(TimedOutTestsListener.java:65) at app//org.junit.runner.notification.SynchronizedRunListener.testFailure(SynchronizedRunListener.java:94) at app//org.junit.runner.notification.RunNotifier$6.notifyListener(RunNotifier.java:177) at app//org.junit.runner.notification.RunNotifier$SafeNotifier.run(RunNotifier.java:72) at app//org.junit.runner.notification.RunNotifier.fireTestFailures(RunNotifier.java:173) at app//org.junit.runner.notification.RunNotifier.fireTestFailure(RunNotifier.java:167) at app//org.apache.maven.surefire.common.junit4.Notifier.fireTestFailure(Notifier.java:100) at app//org.junit.internal.runners.model.EachTestNotifier.addFailure(EachTestNotifier.java:23) at app//org.junit.internal.runners.model.EachTestNotifier.addMultipleFailureException(EachTestNotifier.java:29) at app//org.junit.internal.runners.model.EachTestNotifier.addFailure(EachTestNotifier.java:21) at app//org.junit.runners.ParentRunner.run(ParentRunner.java:419) at app//org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:316) at app//org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:240) at app//org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:214) at app//org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:155) at app//org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:385) at app//org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162) at app//org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:507) at app//org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:495) "HBase-Metrics2-1" daemon prio=5 tid=153 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test.LruBlockCache.EvictionThread" daemon prio=5 tid=192 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:957) "MobFileCache #0" daemon prio=5 tid=196 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS:0;d462fa545078:39401" daemon prio=5 tid=207 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:81) at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:64) at app//org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:906) at app//org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at app//org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) at java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base@17.0.11/javax.security.auth.Subject.doAs(Subject.java:376) at app//org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at app//org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at app//org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server Responder" daemon prio=5 tid=97 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733) at app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716) "ReadOnlyZKClient-127.0.0.1:50115@0x58990532" daemon prio=5 tid=407 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x3939f0b6" daemon prio=5 tid=735 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS_OPEN_META-regionserver/d462fa545078:0-0" daemon prio=5 tid=295 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "FSEditLogAsync" daemon prio=5 tid=53 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:241) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:250) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x11ab1754" daemon prio=5 tid=672 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "master:store-WAL-Roller" daemon prio=5 tid=226 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) "Time-limited test-SendThread(127.0.0.1:50115)" daemon prio=5 tid=189 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) "RPCClient-NioEventLoopGroup-4-5" daemon prio=5 tid=309 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x0a7f1dae" daemon prio=5 tid=567 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "SSL Certificates Store Monitor" daemon prio=5 tid=25 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.Object.wait(Object.java:338) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "master/d462fa545078:0:becomeActiveMaster-HFileCleaner.large.0-1733959174899" daemon prio=5 tid=265 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:101) at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner.consumerLoop(HFileCleaner.java:285) at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner$1.run(HFileCleaner.java:254) "ReadOnlyZKClient-127.0.0.1:50115@0x5af69c10" daemon prio=5 tid=610 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x2ec3b04d" daemon prio=5 tid=543 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x2f8d7d94" daemon prio=5 tid=728 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Socket Reader #1 for port 0" daemon prio=5 tid=55 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497) at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476) "qtp1228916748-40" daemon prio=5 tid=40 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007fd69842a550.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x3a52a7ce" daemon prio=5 tid=440 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "surefire-forkedjvm-command-thread" daemon prio=5 tid=18 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230) at java.base@17.0.11/sun.nio.ch.PendingFuture.get(PendingFuture.java:178) at app//org.apache.maven.surefire.api.util.internal.Channels$2.read(Channels.java:127) at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) at app//org.apache.maven.surefire.api.util.internal.Channels$3.readImpl(Channels.java:169) at app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleReadableChannel.read(AbstractNoninterruptibleReadableChannel.java:50) at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.read(AbstractStreamDecoder.java:430) at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.read(AbstractStreamDecoder.java:419) at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.readMessageType(AbstractStreamDecoder.java:116) at app//org.apache.maven.surefire.booter.stream.CommandDecoder.decode(CommandDecoder.java:77) at app//org.apache.maven.surefire.booter.spi.CommandChannelDecoder.decode(CommandChannelDecoder.java:60) at app//org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:290) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "LruBlockCacheStatsExecutor" daemon prio=5 tid=194 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "HMaster-EventLoopGroup-1-2" daemon prio=10 tid=260 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39401" daemon prio=5 tid=199 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "ReadOnlyZKClient-127.0.0.1:50115@0x0db74723" daemon prio=5 tid=436 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-1" daemon prio=5 tid=303 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x456005b4" daemon prio=5 tid=586 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "master/d462fa545078:0:becomeActiveMaster-HFileCleaner.small.0-1733959174900" daemon prio=5 tid=267 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner.consumerLoop(HFileCleaner.java:285) at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner$2.run(HFileCleaner.java:269) "Hadoop-Metrics-Updater-0" daemon prio=5 tid=58 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 1 on default port 36649" daemon prio=5 tid=65 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "RequestThrottler" daemon prio=5 tid=144 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147) "NIOWorkerThread-10" daemon prio=5 tid=166 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PacketResponder: BP-1905828385-172.17.0.2-1733959167787:blk_1073741837_1013, type=LAST_IN_PIPELINE" daemon prio=5 tid=327 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.Object.wait(Object.java:338) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Idle-Rpc-Conn-Sweeper-pool-0" daemon prio=5 tid=244 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Reference Handler" daemon prio=10 tid=2 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/java.lang.ref.Reference.waitForReferencePendingList(Native Method) at java.base@17.0.11/java.lang.ref.Reference.processPendingReferences(Reference.java:253) at java.base@17.0.11/java.lang.ref.Reference$ReferenceHandler.run(Reference.java:215) "Finalizer" daemon prio=8 tid=3 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) at java.base@17.0.11/java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:172) "IPC Server handler 4 on default port 35819" daemon prio=5 tid=108 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "MarkedDeleteBlockScrubberThread" daemon prio=5 tid=48 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$MarkedDeleteBlockScrubber.run(BlockManager.java:5326) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "JvmPauseMonitor" daemon prio=5 tid=273 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hbase.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:148) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x1c2bec11" daemon prio=5 tid=355 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x7ff9e568" daemon prio=5 tid=577 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x73b6aece" daemon prio=5 tid=335 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x532806a7" daemon prio=5 tid=698 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "pool-5-thread-1" prio=5 tid=36 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "nioEventLoopGroup-2-1" prio=10 tid=92 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) at app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x03f58343" daemon prio=5 tid=510 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x2b7f1575" daemon prio=5 tid=453 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "java.util.concurrent.ThreadPoolExecutor$Worker@45882f1[State = -1, empty queue]" daemon prio=5 tid=135 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "MiniHBaseClusterRegionServer-EventLoopGroup-3-1" daemon prio=10 tid=187 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1228916748-43" daemon prio=5 tid=43 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "pool-11-thread-1" prio=5 tid=69 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x7b00c8de" daemon prio=5 tid=617 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x5df29fae" daemon prio=5 tid=563 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test" daemon prio=5 tid=22 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1864) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1898) at java.base@17.0.11/java.util.concurrent.CompletableFuture.join(CompletableFuture.java:2117) at app//org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi.clearPeerAndQueues(TestAsyncReplicationAdminApi.java:97) at java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) at app//org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at app//org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at app//org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at app//org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at app//org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at app//org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at app//org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at app//org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at app//org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at app//org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at app//org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at app//org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at app//org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at app//org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at app//org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at app//org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at app//org.junit.runners.ParentRunner.run(ParentRunner.java:413) at app//org.junit.runners.Suite.runChild(Suite.java:128) at app//org.junit.runners.Suite.runChild(Suite.java:27) at app//org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at app//org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at app//org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at app//org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at app//org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at app//org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at app//org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at app//org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at app//org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/cluster_e4efebb6-834a-ad19-6c5c-52938933ef0f/data/data2)" daemon prio=5 tid=118 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656) "IPC Server Responder" daemon prio=5 tid=57 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733) at app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716) "NIOServerCxnFactory.SelectorThread-1" daemon prio=5 tid=139 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368) "IPC Server listener on 0" daemon prio=5 tid=94 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559) "ReadOnlyZKClient-127.0.0.1:50115@0x45665e3f" daemon prio=5 tid=417 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PEWorker-2" daemon prio=5 tid=248 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) "Async-Client-Retry-Timer-pool-0" daemon prio=5 tid=242 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RegionServerTracker-0" daemon prio=5 tid=270 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x6ddadebd" daemon prio=5 tid=688 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test.named-queue-events-pool-0" daemon prio=5 tid=186 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) at app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) at app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) at app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x6cab901a" daemon prio=5 tid=507 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1228916748-41-acceptor-0@6900b80-ServerConnector@1af3d2d7{HTTP/1.1, (http/1.1)}{localhost:39787}" daemon prio=3 tid=41 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x59b55eed" daemon prio=5 tid=483 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x0d4f7d2b" daemon prio=5 tid=600 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-6" daemon prio=5 tid=310 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "master/d462fa545078:0.Chore.1" daemon prio=5 tid=262 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 4 on default port 36649" daemon prio=5 tid=68 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "MiniHBaseClusterRegionServer-EventLoopGroup-3-2" daemon prio=10 tid=235 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=39401" daemon prio=5 tid=203 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "LeaseRenewer:jenkins.hfs.0@localhost:36649" daemon prio=5 tid=284 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=41885" daemon prio=5 tid=180 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "DataXceiver for client DFSClient_NONMAPREDUCE_365039782_22 at /127.0.0.1:40862 [Receiving block BP-1905828385-172.17.0.2-1733959167787:blk_1073741837_1013]" daemon prio=5 tid=326 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "SnapshotHandlerChoreCleaner" daemon prio=5 tid=255 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PacketResponder: BP-1905828385-172.17.0.2-1733959167787:blk_1073741843_1019, type=LAST_IN_PIPELINE" daemon prio=5 tid=890 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.Object.wait(Object.java:338) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39401" daemon prio=5 tid=200 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "ProcExecTimeout" daemon prio=5 tid=245 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:279) at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) at app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56) "DatanodeAdminMonitor-0" daemon prio=5 tid=62 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x2a9c6ed6" daemon prio=5 tid=553 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-9" daemon prio=5 tid=165 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Monitor thread for TaskMonitor" daemon prio=5 tid=210 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "MiniHBaseClusterRegionServer-EventLoopGroup-3-3" daemon prio=10 tid=294 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x3a3942de" daemon prio=5 tid=463 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "regionserver/d462fa545078:0.logRoller" daemon prio=5 tid=277 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) "Command processor" daemon prio=5 tid=101 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411) at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) "VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/cluster_e4efebb6-834a-ad19-6c5c-52938933ef0f/data/data1)" daemon prio=5 tid=117 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656) "refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b97db42-8d7d-e1c6-8679-f75c215b7141/cluster_e4efebb6-834a-ad19-6c5c-52938933ef0f/data/data2/current/BP-1905828385-172.17.0.2-1733959167787" daemon prio=5 tid=124 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=41885" daemon prio=5 tid=181 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "ReadOnlyZKClient-127.0.0.1:50115@0x52af179f" daemon prio=5 tid=487 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 1 on default port 35819" daemon prio=5 tid=105 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "RPCClient-NioEventLoopGroup-4-4" daemon prio=5 tid=308 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1228916748-42" daemon prio=5 tid=42 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-4" daemon prio=5 tid=159 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x3eff4389" daemon prio=5 tid=648 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "BP-1905828385-172.17.0.2-1733959167787 heartbeating to localhost/127.0.0.1:36649" daemon prio=5 tid=102 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:771) at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:914) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp971800375-90" daemon prio=5 tid=90 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=39401" daemon prio=5 tid=206 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=5 tid=23 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) at app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Common-Cleaner" daemon prio=8 tid=12 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@17.0.11/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:140) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) at java.base@17.0.11/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:162) "RpcClient-timer-pool-0" daemon prio=5 tid=243 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@4775fd9b" daemon prio=5 tid=49 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:563) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "DataXceiver for client DFSClient_NONMAPREDUCE_365039782_22 at /127.0.0.1:40818 [Receiving block BP-1905828385-172.17.0.2-1733959167787:blk_1073741832_1008]" daemon prio=5 tid=282 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "normalizer-worker-0" daemon prio=5 tid=254 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorkQueue.take(RegionNormalizerWorkQueue.java:146) at app//org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorker.run(RegionNormalizerWorker.java:191) at java.base@17.0.11/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "OldWALsCleaner-0" daemon prio=5 tid=261 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner.deleteFile(LogCleaner.java:172) at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner.lambda$createOldWalsCleaner$1(LogCleaner.java:152) at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner$$Lambda$610/0x00007fd698af75d0.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41885" daemon prio=5 tid=177 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "pool-19-thread-1" prio=5 tid=103 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x1507cc71" daemon prio=5 tid=523 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x3b121ec8" daemon prio=5 tid=503 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x5e1b5946" daemon prio=5 tid=596 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 2 on default port 35819" daemon prio=5 tid=106 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "ReadOnlyZKClient-127.0.0.1:50115@0x37052b90" daemon prio=5 tid=423 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor@369c8982" daemon prio=5 tid=61 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor.run(PendingReconstructionBlocks.java:267) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-5" daemon prio=5 tid=161 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Session-HouseKeeper-180050c4-1" prio=5 tid=45 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-2" daemon prio=5 tid=157 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x3518cbae" daemon prio=5 tid=351 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "BootstrapNodeManager" daemon prio=5 tid=258 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS_COMPACTED_FILES_DISCHARGER-regionserver/d462fa545078:0-0" daemon prio=5 tid=749 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOServerCxnFactory.SelectorThread-0" daemon prio=5 tid=138 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368) "RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39401" daemon prio=5 tid=197 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41885" daemon prio=5 tid=179 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "org.apache.hadoop.util.JvmPauseMonitor$Monitor@5f849ed1" daemon prio=5 tid=93 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x7730277e" daemon prio=5 tid=446 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "AsyncFSWAL-0-hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187-prefix:d462fa545078,39401,1733959172007.meta" daemon prio=5 tid=298 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "zk-event-processor-pool-0" daemon prio=5 tid=191 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "AsyncFSWAL-0-hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187/MasterData-prefix:d462fa545078,41885,1733959171253" daemon prio=5 tid=238 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS:0;d462fa545078:39401-longCompactions-0" daemon prio=5 tid=274 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:101) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "pool-1-thread-2" daemon prio=5 tid=15 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.poll(EPollPort.java:200) at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.run(EPollPort.java:281) at java.base@17.0.11/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:113) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOServerCxnFactory.AcceptThread:localhost/127.0.0.1:50115" daemon prio=5 tid=140 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181) "ReadOnlyZKClient-127.0.0.1:50115@0x1732b7ea" daemon prio=5 tid=497 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "FsDatasetAsyncDiskServiceFixer" daemon prio=5 tid=136 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer.run(HBaseTestingUtil.java:576) "ReadOnlyZKClient-127.0.0.1:50115@0x0a69c86a" daemon prio=5 tid=341 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x48659abb" daemon prio=5 tid=732 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@740e026a" daemon prio=5 tid=85 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) at java.base@17.0.11/sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:126) at app//org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:242) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.util.JvmPauseMonitor$Monitor@7679e392" daemon prio=5 tid=34 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.PeerCache@69fcf8a7" daemon prio=5 tid=216 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) at app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x766cde9a" daemon prio=5 tid=678 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1228916748-38" daemon prio=5 tid=38 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007fd69842a550.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@6a7b800b" daemon prio=5 tid=71 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:537) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-12" daemon prio=5 tid=168 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "master/d462fa545078:0:becomeActiveMaster-MemStoreChunkPool Statistics" daemon prio=5 tid=212 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-11" daemon prio=5 tid=167 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "master:store-Flusher" daemon prio=5 tid=241 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759) at app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor.flushLoop(MasterRegionFlusherAndCompactor.java:200) at app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor$$Lambda$481/0x00007fd698a1ea18.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=41885" daemon prio=5 tid=183 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "ReadOnlyZKClient-127.0.0.1:50115@0x6e04bf1b" daemon prio=5 tid=613 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x24fd53e0" daemon prio=5 tid=631 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@75d00e59" daemon prio=5 tid=73 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:4592) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test-EventThread" daemon prio=5 tid=156 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) "IPC Server idle connection scanner for port 36649" daemon prio=5 tid=56 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "LeaseRenewer:jenkins@localhost:36649" daemon prio=5 tid=147 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x730cd2c1" daemon prio=5 tid=361 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x3c55c34c" daemon prio=5 tid=603 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RedundancyMonitor" daemon prio=5 tid=47 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at java.base@17.0.11/java.lang.Thread.sleep(Thread.java:344) at java.base@17.0.11/java.util.concurrent.TimeUnit.sleep(TimeUnit.java:446) at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$RedundancyMonitor.run(BlockManager.java:5352) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 0 on default port 35819" daemon prio=5 tid=104 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "ReadOnlyZKClient-127.0.0.1:50115@0x6a54443c" daemon prio=5 tid=722 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=39401" daemon prio=5 tid=204 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "Time-limited test-EventThread" daemon prio=5 tid=190 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) "AsyncFSWAL-0-hdfs://localhost:36649/user/jenkins/test-data/a83fc26f-8a05-128b-9608-99cbe1813187-prefix:d462fa545078,39401,1733959172007" daemon prio=5 tid=285 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x64d2e993" daemon prio=5 tid=712 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x59b483b6" daemon prio=5 tid=533 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-13" daemon prio=5 tid=169 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x4bdc13ef" daemon prio=5 tid=557 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-1" daemon prio=5 tid=145 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Registry-endpoints-refresh-end-points" daemon prio=5 tid=307 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.client.RegistryEndpointsRefresher.mainLoop(RegistryEndpointsRefresher.java:92) at app//org.apache.hadoop.hbase.client.RegistryEndpointsRefresher$$Lambda$806/0x00007fd698bb6410.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x761e40f2" daemon prio=5 tid=398 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-16" daemon prio=5 tid=172 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x186f86f5" daemon prio=5 tid=590 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x3b144df5" daemon prio=5 tid=547 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Client (1933853537) connection to localhost/127.0.0.1:36649 from jenkins" daemon prio=5 tid=109 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) at app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) "Hadoop-Metrics-Updater-0" daemon prio=5 tid=98 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PacketResponder: BP-1905828385-172.17.0.2-1733959167787:blk_1073741834_1010, type=LAST_IN_PIPELINE" daemon prio=5 tid=297 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.Object.wait(Object.java:338) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-2" daemon prio=5 tid=305 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Timer for 'HBase' metrics system" daemon prio=5 tid=1006 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "regionserver/d462fa545078:0.Chore.1" daemon prio=5 tid=275 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PacketResponder: BP-1905828385-172.17.0.2-1733959167787:blk_1073741832_1008, type=LAST_IN_PIPELINE" daemon prio=5 tid=283 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.Object.wait(Object.java:338) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50115@0x2de668f0" daemon prio=5 tid=395 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-3" daemon prio=5 tid=158 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PEWorker-1" daemon prio=5 tid=247 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) "Signal Dispatcher" daemon prio=9 tid=4 runnable java.lang.Thread.State: RUNNABLE "ReadOnlyZKClient-127.0.0.1:50115@0x4b36911b" daemon prio=5 tid=345 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007fd698be6b58.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100" daemon prio=5 tid=35 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)