2024-12-06 15:40:50,052 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1be2019a 2024-12-06 15:40:50,069 main DEBUG Took 0.014793 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-06 15:40:50,069 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-06 15:40:50,071 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-06 15:40:50,072 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-06 15:40:50,073 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:40:50,082 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-06 15:40:50,106 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,108 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:40:50,109 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,109 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:40:50,110 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,110 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:40:50,111 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,112 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:40:50,112 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,113 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:40:50,114 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,114 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:40:50,115 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,115 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:40:50,116 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,116 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:40:50,117 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,117 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:40:50,118 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,118 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:40:50,119 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,120 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:40:50,120 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,121 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:40:50,121 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,122 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-06 15:40:50,123 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:40:50,126 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-06 15:40:50,128 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-06 15:40:50,128 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-06 15:40:50,130 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-06 15:40:50,131 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-06 15:40:50,145 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-06 15:40:50,151 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-06 15:40:50,153 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-06 15:40:50,154 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-06 15:40:50,154 main DEBUG createAppenders(={Console}) 2024-12-06 15:40:50,155 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1be2019a initialized 2024-12-06 15:40:50,156 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1be2019a 2024-12-06 15:40:50,156 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1be2019a OK. 2024-12-06 15:40:50,157 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-06 15:40:50,157 main DEBUG OutputStream closed 2024-12-06 15:40:50,158 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-06 15:40:50,158 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-06 15:40:50,159 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@47d9a273 OK 2024-12-06 15:40:50,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-06 15:40:50,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-06 15:40:50,269 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-06 15:40:50,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-06 15:40:50,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-06 15:40:50,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-06 15:40:50,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-06 15:40:50,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-06 15:40:50,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-06 15:40:50,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-06 15:40:50,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-06 15:40:50,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-06 15:40:50,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-06 15:40:50,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-06 15:40:50,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-06 15:40:50,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-06 15:40:50,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-06 15:40:50,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-06 15:40:50,280 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06 15:40:50,280 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@177bea38) with optional ClassLoader: null 2024-12-06 15:40:50,281 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-06 15:40:50,281 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@177bea38] started OK. 2024-12-06T15:40:50,300 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.wal.TestWALFactory timeout: 13 mins 2024-12-06 15:40:50,305 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-06 15:40:50,305 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06T15:40:50,787 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d 2024-12-06T15:40:50,822 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225, deleteOnExit=true 2024-12-06T15:40:50,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/test.cache.data in system properties and HBase conf 2024-12-06T15:40:50,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.tmp.dir in system properties and HBase conf 2024-12-06T15:40:50,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir in system properties and HBase conf 2024-12-06T15:40:50,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-06T15:40:50,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-06T15:40:50,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-06T15:40:50,953 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-06T15:40:51,074 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T15:40:51,086 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-06T15:40:51,087 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-06T15:40:51,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-06T15:40:51,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T15:40:51,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-06T15:40:51,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-06T15:40:51,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T15:40:51,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T15:40:51,104 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-06T15:40:51,105 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/nfs.dump.dir in system properties and HBase conf 2024-12-06T15:40:51,105 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/java.io.tmpdir in system properties and HBase conf 2024-12-06T15:40:51,105 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T15:40:51,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-06T15:40:51,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-06T15:40:51,773 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-06T15:40:52,180 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-06T15:40:52,284 INFO [Time-limited test {}] log.Log(170): Logging initialized @3521ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-06T15:40:52,369 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:40:52,446 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T15:40:52,477 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T15:40:52,478 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T15:40:52,479 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T15:40:52,492 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:40:52,495 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70940934{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,AVAILABLE} 2024-12-06T15:40:52,496 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d3e0f30{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T15:40:52,730 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@179dea64{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/java.io.tmpdir/jetty-localhost-39425-hadoop-hdfs-3_4_1-tests_jar-_-any-327312351224586494/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T15:40:52,737 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7975eec2{HTTP/1.1, (http/1.1)}{localhost:39425} 2024-12-06T15:40:52,738 INFO [Time-limited test {}] server.Server(415): Started @3976ms 2024-12-06T15:40:52,796 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-06T15:40:53,190 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:40:53,198 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T15:40:53,199 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T15:40:53,199 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T15:40:53,200 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T15:40:53,201 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6cfcc17f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,AVAILABLE} 2024-12-06T15:40:53,202 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6034ceb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T15:40:53,333 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28de11c6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/java.io.tmpdir/jetty-localhost-32935-hadoop-hdfs-3_4_1-tests_jar-_-any-2975352061076648147/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:40:53,334 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@52cb6d33{HTTP/1.1, (http/1.1)}{localhost:32935} 2024-12-06T15:40:53,334 INFO [Time-limited test {}] server.Server(415): Started @4573ms 2024-12-06T15:40:53,394 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T15:40:53,531 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:40:53,538 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T15:40:53,542 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T15:40:53,542 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T15:40:53,543 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T15:40:53,544 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@997ee7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,AVAILABLE} 2024-12-06T15:40:53,545 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62756a49{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T15:40:53,714 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7f751333{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/java.io.tmpdir/jetty-localhost-45451-hadoop-hdfs-3_4_1-tests_jar-_-any-3981988810761554151/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:40:53,716 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@502117d{HTTP/1.1, (http/1.1)}{localhost:45451} 2024-12-06T15:40:53,716 INFO [Time-limited test {}] server.Server(415): Started @4955ms 2024-12-06T15:40:53,719 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T15:40:53,916 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:40:53,928 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T15:40:53,942 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T15:40:53,942 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T15:40:53,943 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T15:40:53,946 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef440a4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,AVAILABLE} 2024-12-06T15:40:53,947 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13eb518c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T15:40:54,010 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data3/current/BP-1829792084-172.17.0.2-1733499651885/current, will proceed with Du for space computation calculation, 2024-12-06T15:40:54,012 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data1/current/BP-1829792084-172.17.0.2-1733499651885/current, will proceed with Du for space computation calculation, 2024-12-06T15:40:54,028 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data4/current/BP-1829792084-172.17.0.2-1733499651885/current, will proceed with Du for space computation calculation, 2024-12-06T15:40:54,032 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data2/current/BP-1829792084-172.17.0.2-1733499651885/current, will proceed with Du for space computation calculation, 2024-12-06T15:40:54,142 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dcc801f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/java.io.tmpdir/jetty-localhost-46151-hadoop-hdfs-3_4_1-tests_jar-_-any-11694917044096353568/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:40:54,144 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7c3ad891{HTTP/1.1, (http/1.1)}{localhost:46151} 2024-12-06T15:40:54,144 INFO [Time-limited test {}] server.Server(415): Started @5383ms 2024-12-06T15:40:54,151 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T15:40:54,152 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T15:40:54,166 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T15:40:54,246 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3d6937db3bcada12 with lease ID 0xf67832b37efb77b5: Processing first storage report for DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36 from datanode DatanodeRegistration(127.0.0.1:36541, datanodeUuid=c262f2ad-b09a-4750-9e9a-43ee971188b0, infoPort=36457, infoSecurePort=0, ipcPort=37691, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:40:54,248 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3d6937db3bcada12 with lease ID 0xf67832b37efb77b5: from storage DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36 node DatanodeRegistration(127.0.0.1:36541, datanodeUuid=c262f2ad-b09a-4750-9e9a-43ee971188b0, infoPort=36457, infoSecurePort=0, ipcPort=37691, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-06T15:40:54,249 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4d96523c44bf9331 with lease ID 0xf67832b37efb77b6: Processing first storage report for DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6 from datanode DatanodeRegistration(127.0.0.1:46587, datanodeUuid=466c1d1c-bc74-4274-ab94-ebed5ce6b316, infoPort=41695, infoSecurePort=0, ipcPort=37279, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:40:54,249 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4d96523c44bf9331 with lease ID 0xf67832b37efb77b6: from storage DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6 node DatanodeRegistration(127.0.0.1:46587, datanodeUuid=466c1d1c-bc74-4274-ab94-ebed5ce6b316, infoPort=41695, infoSecurePort=0, ipcPort=37279, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-06T15:40:54,249 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3d6937db3bcada12 with lease ID 0xf67832b37efb77b5: Processing first storage report for DS-f2cff151-b014-48d5-92f8-12da8f3c2838 from datanode DatanodeRegistration(127.0.0.1:36541, datanodeUuid=c262f2ad-b09a-4750-9e9a-43ee971188b0, infoPort=36457, infoSecurePort=0, ipcPort=37691, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:40:54,250 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3d6937db3bcada12 with lease ID 0xf67832b37efb77b5: from storage DS-f2cff151-b014-48d5-92f8-12da8f3c2838 node DatanodeRegistration(127.0.0.1:36541, datanodeUuid=c262f2ad-b09a-4750-9e9a-43ee971188b0, infoPort=36457, infoSecurePort=0, ipcPort=37691, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T15:40:54,255 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4d96523c44bf9331 with lease ID 0xf67832b37efb77b6: Processing first storage report for DS-67b24bf5-c01f-4c33-a833-8f1c68f972e8 from datanode DatanodeRegistration(127.0.0.1:46587, datanodeUuid=466c1d1c-bc74-4274-ab94-ebed5ce6b316, infoPort=41695, infoSecurePort=0, ipcPort=37279, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:40:54,255 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4d96523c44bf9331 with lease ID 0xf67832b37efb77b6: from storage DS-67b24bf5-c01f-4c33-a833-8f1c68f972e8 node DatanodeRegistration(127.0.0.1:46587, datanodeUuid=466c1d1c-bc74-4274-ab94-ebed5ce6b316, infoPort=41695, infoSecurePort=0, ipcPort=37279, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T15:40:54,337 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data5/current/BP-1829792084-172.17.0.2-1733499651885/current, will proceed with Du for space computation calculation, 2024-12-06T15:40:54,338 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data6/current/BP-1829792084-172.17.0.2-1733499651885/current, will proceed with Du for space computation calculation, 2024-12-06T15:40:54,377 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T15:40:54,383 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x825386fbefcbc31 with lease ID 0xf67832b37efb77b7: Processing first storage report for DS-4f4b3052-a29f-49e8-a992-229f8202a569 from datanode DatanodeRegistration(127.0.0.1:35529, datanodeUuid=0d6e69cc-7877-4434-b2cb-608913e6b424, infoPort=37899, infoSecurePort=0, ipcPort=36721, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:40:54,383 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x825386fbefcbc31 with lease ID 0xf67832b37efb77b7: from storage DS-4f4b3052-a29f-49e8-a992-229f8202a569 node DatanodeRegistration(127.0.0.1:35529, datanodeUuid=0d6e69cc-7877-4434-b2cb-608913e6b424, infoPort=37899, infoSecurePort=0, ipcPort=36721, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T15:40:54,383 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x825386fbefcbc31 with lease ID 0xf67832b37efb77b7: Processing first storage report for DS-08640c80-06d2-4c05-ad8c-67c9d4028b0c from datanode DatanodeRegistration(127.0.0.1:35529, datanodeUuid=0d6e69cc-7877-4434-b2cb-608913e6b424, infoPort=37899, infoSecurePort=0, ipcPort=36721, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:40:54,384 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x825386fbefcbc31 with lease ID 0xf67832b37efb77b7: from storage DS-08640c80-06d2-4c05-ad8c-67c9d4028b0c node DatanodeRegistration(127.0.0.1:35529, datanodeUuid=0d6e69cc-7877-4434-b2cb-608913e6b424, infoPort=37899, infoSecurePort=0, ipcPort=36721, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T15:40:54,660 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d 2024-12-06T15:40:54,678 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T15:40:54,682 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T15:40:55,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741825_1001 (size=7) 2024-12-06T15:40:55,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741825_1001 (size=7) 2024-12-06T15:40:55,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741825_1001 (size=7) 2024-12-06T15:40:55,411 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37105/user/jenkins/test-data/8b6337b1-183f-37ae-d1f3-fef1d3c371dc with version=8 2024-12-06T15:40:55,412 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T15:40:55,416 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T15:40:55,435 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testVisitors Thread=159, OpenFileDescriptor=401, MaxFileDescriptor=1048576, SystemLoadAverage=418, ProcessCount=11, AvailableMemoryMB=10529 2024-12-06T15:40:55,540 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:214) ~[classes/:?] at org.apache.hadoop.hbase.wal.TestWALFactory.setUp(TestWALFactory.java:123) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:40:55,545 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:55,551 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-06T15:40:55,568 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-06T15:40:55,568 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-06T15:40:55,625 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-06T15:40:55,634 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-06T15:40:55,646 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T15:40:55,673 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-12-06T15:40:55,682 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=testvisitors%2C16010%2C1, suffix=, logDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testvisitors,16010,1, archiveDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs, maxLogs=1760 2024-12-06T15:40:55,757 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testvisitors,16010,1/testvisitors%2C16010%2C1.1733499655725, exclude list is [], retry=0 2024-12-06T15:40:55,779 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1558 (auto-detected) 2024-12-06T15:40:55,784 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-06T15:40:55,817 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:55,817 DEBUG [AsyncFSWAL-1-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:55,819 DEBUG [AsyncFSWAL-1-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:55,822 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-06T15:40:55,912 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testvisitors,16010,1/testvisitors%2C16010%2C1.1733499655725 2024-12-06T15:40:55,914 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36457:36457),(127.0.0.1/127.0.0.1:41695:41695),(127.0.0.1/127.0.0.1:37899:37899)] 2024-12-06T15:40:55,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741826_1002 (size=1271) 2024-12-06T15:40:55,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741826_1002 (size=1271) 2024-12-06T15:40:55,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741826_1002 (size=1271) 2024-12-06T15:40:55,985 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs 2024-12-06T15:40:55,990 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL testvisitors%2C16010%2C1:(num 1733499655725) 2024-12-06T15:40:56,020 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testVisitors Thread=164 (was 159) Potentially hanging thread: AsyncFSWAL-1-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-1-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-1-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=433 (was 401) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=418 (was 418), ProcessCount=11 (was 11), AvailableMemoryMB=10489 (was 10529) 2024-12-06T15:40:56,030 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testSplit Thread=164, OpenFileDescriptor=433, MaxFileDescriptor=1048576, SystemLoadAverage=418, ProcessCount=11, AvailableMemoryMB=10487 2024-12-06T15:40:56,031 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:56,041 INFO [Time-limited test {}] wal.TestWALFactory(199): allo hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd 2024-12-06T15:40:56,045 INFO [Time-limited test {}] wal.TestWALFactory(199): allo hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7 2024-12-06T15:40:56,048 INFO [Time-limited test {}] wal.TestWALFactory(199): allo hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a 2024-12-06T15:40:56,048 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T15:40:56,054 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-12-06T15:40:56,054 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=testsplit%2C16010%2C1, suffix=, logDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1, archiveDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs, maxLogs=1760 2024-12-06T15:40:56,070 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055, exclude list is [], retry=0 2024-12-06T15:40:56,077 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:56,079 DEBUG [AsyncFSWAL-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:56,086 DEBUG [AsyncFSWAL-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:56,104 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055 2024-12-06T15:40:56,105 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:41695:41695),(127.0.0.1/127.0.0.1:36457:36457)] 2024-12-06T15:40:56,111 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 0: [#edits: 1 = ] 2024-12-06T15:40:56,112 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 0: [#edits: 1 = ] 2024-12-06T15:40:56,113 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 0: [#edits: 1 = ] 2024-12-06T15:40:56,130 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116, exclude list is [], retry=0 2024-12-06T15:40:56,137 DEBUG [AsyncFSWAL-3-4 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:56,138 DEBUG [AsyncFSWAL-3-6 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:56,138 DEBUG [AsyncFSWAL-3-5 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:56,147 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055 with entries=3, filesize=436 B; new WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116 2024-12-06T15:40:56,148 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36457:36457),(127.0.0.1/127.0.0.1:41695:41695),(127.0.0.1/127.0.0.1:37899:37899)] 2024-12-06T15:40:56,148 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055 is not closed yet, will try archiving it next time 2024-12-06T15:40:56,149 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 1: [#edits: 1 = ] 2024-12-06T15:40:56,149 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 1: [#edits: 1 = ] 2024-12-06T15:40:56,149 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 1: [#edits: 1 = ] 2024-12-06T15:40:56,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741827_1003 (size=444) 2024-12-06T15:40:56,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741827_1003 (size=444) 2024-12-06T15:40:56,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741827_1003 (size=444) 2024-12-06T15:40:56,172 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156, exclude list is [], retry=0 2024-12-06T15:40:56,177 DEBUG [AsyncFSWAL-3-7 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:56,178 DEBUG [AsyncFSWAL-3-8 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:56,178 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:56,185 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116 with entries=3, filesize=436 B; new WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156 2024-12-06T15:40:56,186 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41695:41695),(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:36457:36457)] 2024-12-06T15:40:56,186 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116 is not closed yet, will try archiving it next time 2024-12-06T15:40:56,186 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 2: [#edits: 1 = ] 2024-12-06T15:40:56,187 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 2: [#edits: 1 = ] 2024-12-06T15:40:56,187 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 2: [#edits: 1 = ] 2024-12-06T15:40:56,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741828_1004 (size=444) 2024-12-06T15:40:56,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741828_1004 (size=444) 2024-12-06T15:40:56,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741828_1004 (size=444) 2024-12-06T15:40:56,217 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193, exclude list is [], retry=0 2024-12-06T15:40:56,235 DEBUG [AsyncFSWAL-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:56,236 DEBUG [AsyncFSWAL-3-4 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:56,235 DEBUG [AsyncFSWAL-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:56,249 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156 with entries=3, filesize=436 B; new WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193 2024-12-06T15:40:56,250 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:36457:36457),(127.0.0.1/127.0.0.1:41695:41695)] 2024-12-06T15:40:56,250 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156 is not closed yet, will try archiving it next time 2024-12-06T15:40:56,251 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 0: [#edits: 1 = ] 2024-12-06T15:40:56,251 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 0: [#edits: 1 = ] 2024-12-06T15:40:56,251 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 0: [#edits: 1 = ] 2024-12-06T15:40:56,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741829_1005 (size=444) 2024-12-06T15:40:56,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741829_1005 (size=444) 2024-12-06T15:40:56,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741829_1005 (size=444) 2024-12-06T15:40:56,277 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259, exclude list is [], retry=0 2024-12-06T15:40:56,284 DEBUG [AsyncFSWAL-3-5 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:56,284 DEBUG [AsyncFSWAL-3-7 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:56,284 DEBUG [AsyncFSWAL-3-6 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:56,291 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193 with entries=3, filesize=436 B; new WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259 2024-12-06T15:40:56,291 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41695:41695),(127.0.0.1/127.0.0.1:36457:36457),(127.0.0.1/127.0.0.1:37899:37899)] 2024-12-06T15:40:56,292 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193 is not closed yet, will try archiving it next time 2024-12-06T15:40:56,292 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 1: [#edits: 1 = ] 2024-12-06T15:40:56,293 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 1: [#edits: 1 = ] 2024-12-06T15:40:56,293 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 1: [#edits: 1 = ] 2024-12-06T15:40:56,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741830_1006 (size=444) 2024-12-06T15:40:56,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741830_1006 (size=444) 2024-12-06T15:40:56,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741830_1006 (size=444) 2024-12-06T15:40:56,317 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297, exclude list is [], retry=0 2024-12-06T15:40:56,323 DEBUG [AsyncFSWAL-3-8 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:56,323 DEBUG [AsyncFSWAL-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:56,323 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:56,339 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259 with entries=3, filesize=436 B; new WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 2024-12-06T15:40:56,340 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41695:41695),(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:36457:36457)] 2024-12-06T15:40:56,340 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259 is not closed yet, will try archiving it next time 2024-12-06T15:40:56,341 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 2: [#edits: 1 = ] 2024-12-06T15:40:56,341 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 2: [#edits: 1 = ] 2024-12-06T15:40:56,341 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 2: [#edits: 1 = ] 2024-12-06T15:40:56,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741831_1007 (size=444) 2024-12-06T15:40:56,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741831_1007 (size=444) 2024-12-06T15:40:56,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741831_1007 (size=444) 2024-12-06T15:40:56,369 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347, exclude list is [], retry=0 2024-12-06T15:40:56,375 DEBUG [AsyncFSWAL-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:56,375 DEBUG [AsyncFSWAL-3-4 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:56,376 DEBUG [AsyncFSWAL-3-5 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:56,385 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 with entries=3, filesize=436 B; new WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347 2024-12-06T15:40:56,385 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:41695:41695),(127.0.0.1/127.0.0.1:36457:36457)] 2024-12-06T15:40:56,385 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 is not closed yet, will try archiving it next time 2024-12-06T15:40:56,386 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 0: [#edits: 1 = ] 2024-12-06T15:40:56,386 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 0: [#edits: 1 = ] 2024-12-06T15:40:56,387 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 0: [#edits: 1 = ] 2024-12-06T15:40:56,390 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 not finished, retry = 0 2024-12-06T15:40:56,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741832_1008 (size=444) 2024-12-06T15:40:56,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741832_1008 (size=444) 2024-12-06T15:40:56,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741832_1008 (size=444) 2024-12-06T15:40:56,412 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390, exclude list is [], retry=0 2024-12-06T15:40:56,417 DEBUG [AsyncFSWAL-3-6 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:56,417 DEBUG [AsyncFSWAL-3-7 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:56,417 DEBUG [AsyncFSWAL-3-8 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:56,423 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347 with entries=3, filesize=436 B; new WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390 2024-12-06T15:40:56,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:36457:36457),(127.0.0.1/127.0.0.1:41695:41695)] 2024-12-06T15:40:56,424 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 is not closed yet, will try archiving it next time 2024-12-06T15:40:56,424 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347 is not closed yet, will try archiving it next time 2024-12-06T15:40:56,425 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 1: [#edits: 1 = ] 2024-12-06T15:40:56,425 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 1: [#edits: 1 = ] 2024-12-06T15:40:56,426 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 1: [#edits: 1 = ] 2024-12-06T15:40:56,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741833_1009 (size=444) 2024-12-06T15:40:56,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741833_1009 (size=444) 2024-12-06T15:40:56,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741833_1009 (size=444) 2024-12-06T15:40:56,431 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 is not closed yet, will try archiving it next time 2024-12-06T15:40:56,452 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430, exclude list is [], retry=0 2024-12-06T15:40:56,458 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:56,458 DEBUG [AsyncFSWAL-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:56,459 DEBUG [AsyncFSWAL-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:56,470 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390 with entries=3, filesize=436 B; new WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430 2024-12-06T15:40:56,470 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:41695:41695),(127.0.0.1/127.0.0.1:36457:36457)] 2024-12-06T15:40:56,470 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 is not closed yet, will try archiving it next time 2024-12-06T15:40:56,470 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390 is not closed yet, will try archiving it next time 2024-12-06T15:40:56,471 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 2: [#edits: 1 = ] 2024-12-06T15:40:56,471 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 2: [#edits: 1 = ] 2024-12-06T15:40:56,471 INFO [Time-limited test {}] wal.TestWALFactory(215): Region 2: [#edits: 1 = ] 2024-12-06T15:40:56,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741834_1010 (size=444) 2024-12-06T15:40:56,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741834_1010 (size=444) 2024-12-06T15:40:56,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741834_1010 (size=444) 2024-12-06T15:40:56,490 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 is not closed yet, will try archiving it next time 2024-12-06T15:40:56,495 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656475, exclude list is [], retry=0 2024-12-06T15:40:56,500 DEBUG [AsyncFSWAL-3-6 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:56,500 DEBUG [AsyncFSWAL-3-4 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:56,500 DEBUG [AsyncFSWAL-3-5 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:56,504 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430 with entries=3, filesize=436 B; new WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656475 2024-12-06T15:40:56,504 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36457:36457),(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:41695:41695)] 2024-12-06T15:40:56,504 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430 is not closed yet, will try archiving it next time 2024-12-06T15:40:56,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741835_1011 (size=444) 2024-12-06T15:40:56,514 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430 not finished, retry = 0 2024-12-06T15:40:56,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741835_1011 (size=444) 2024-12-06T15:40:56,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741835_1011 (size=444) 2024-12-06T15:40:56,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741836_1012 (size=93) 2024-12-06T15:40:56,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741836_1012 (size=93) 2024-12-06T15:40:56,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741836_1012 (size=93) 2024-12-06T15:40:56,667 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055, size=444 (444bytes) 2024-12-06T15:40:56,668 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-06T15:40:56,668 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-06T15:40:56,668 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055 2024-12-06T15:40:56,675 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055 after 5ms 2024-12-06T15:40:56,720 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055 took 57ms 2024-12-06T15:40:56,728 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055 so closing down 2024-12-06T15:40:56,732 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T15:40:56,757 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-testsplit%2C16010%2C1.1733499656055.temp 2024-12-06T15:40:56,760 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000002-testsplit%2C16010%2C1.1733499656055.temp 2024-12-06T15:40:56,764 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T15:40:56,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741837_1013 (size=444) 2024-12-06T15:40:56,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741837_1013 (size=444) 2024-12-06T15:40:56,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741837_1013 (size=444) 2024-12-06T15:40:56,779 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000002-testsplit%2C16010%2C1.1733499656055.temp (wrote 3 edits, skipped 0 edits in 3 ms) 2024-12-06T15:40:56,782 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000002-testsplit%2C16010%2C1.1733499656055.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000004 2024-12-06T15:40:56,787 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055, size=444, length=444, corrupted=false, cancelled=false 2024-12-06T15:40:56,789 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055, journal: Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055, size=444 (444bytes) at 1733499656667Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055 so closing down at 1733499656728 (+61 ms)Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000002-testsplit%2C16010%2C1.1733499656055.temp at 1733499656760 (+32 ms)3 split writer threads finished at 1733499656765 (+5 ms)Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000002-testsplit%2C16010%2C1.1733499656055.temp (wrote 3 edits, skipped 0 edits in 3 ms) at 1733499656779 (+14 ms)Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000002-testsplit%2C16010%2C1.1733499656055.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000004 at 1733499656782 (+3 ms)Processed 3 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055, size=444, length=444, corrupted=false, cancelled=false at 1733499656787 (+5 ms) 2024-12-06T15:40:56,793 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656055 to hdfs://localhost:37105/user/jenkins/test-data/8b6337b1-183f-37ae-d1f3-fef1d3c371dc/oldWALs/testsplit%2C16010%2C1.1733499656055 2024-12-06T15:40:56,793 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116, size=444 (444bytes) 2024-12-06T15:40:56,793 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116 2024-12-06T15:40:56,794 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116 after 0ms 2024-12-06T15:40:56,799 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116 took 6ms 2024-12-06T15:40:56,802 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116 so closing down 2024-12-06T15:40:56,802 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T15:40:56,806 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-testsplit%2C16010%2C1.1733499656116.temp 2024-12-06T15:40:56,808 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000005-testsplit%2C16010%2C1.1733499656116.temp 2024-12-06T15:40:56,808 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T15:40:56,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741838_1014 (size=444) 2024-12-06T15:40:56,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741838_1014 (size=444) 2024-12-06T15:40:56,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741838_1014 (size=444) 2024-12-06T15:40:56,818 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000005-testsplit%2C16010%2C1.1733499656116.temp (wrote 3 edits, skipped 0 edits in 0 ms) 2024-12-06T15:40:56,821 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000005-testsplit%2C16010%2C1.1733499656116.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000007 2024-12-06T15:40:56,822 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116, size=444, length=444, corrupted=false, cancelled=false 2024-12-06T15:40:56,822 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116, journal: Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116, size=444 (444bytes) at 1733499656793Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116 so closing down at 1733499656802 (+9 ms)Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000005-testsplit%2C16010%2C1.1733499656116.temp at 1733499656808 (+6 ms)3 split writer threads finished at 1733499656808Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000005-testsplit%2C16010%2C1.1733499656116.temp (wrote 3 edits, skipped 0 edits in 0 ms) at 1733499656818 (+10 ms)Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000005-testsplit%2C16010%2C1.1733499656116.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000007 at 1733499656821 (+3 ms)Processed 3 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116, size=444, length=444, corrupted=false, cancelled=false at 1733499656822 (+1 ms) 2024-12-06T15:40:56,826 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656116 to hdfs://localhost:37105/user/jenkins/test-data/8b6337b1-183f-37ae-d1f3-fef1d3c371dc/oldWALs/testsplit%2C16010%2C1.1733499656116 2024-12-06T15:40:56,827 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156, size=444 (444bytes) 2024-12-06T15:40:56,827 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156 2024-12-06T15:40:56,828 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156 after 0ms 2024-12-06T15:40:56,833 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156 took 7ms 2024-12-06T15:40:56,837 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156 so closing down 2024-12-06T15:40:56,837 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T15:40:56,840 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000008-testsplit%2C16010%2C1.1733499656156.temp 2024-12-06T15:40:56,842 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000008-testsplit%2C16010%2C1.1733499656156.temp 2024-12-06T15:40:56,842 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T15:40:56,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741839_1015 (size=444) 2024-12-06T15:40:56,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741839_1015 (size=444) 2024-12-06T15:40:56,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741839_1015 (size=444) 2024-12-06T15:40:56,853 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000008-testsplit%2C16010%2C1.1733499656156.temp (wrote 3 edits, skipped 0 edits in 0 ms) 2024-12-06T15:40:56,855 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000008-testsplit%2C16010%2C1.1733499656156.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000010 2024-12-06T15:40:56,855 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156, size=444, length=444, corrupted=false, cancelled=false 2024-12-06T15:40:56,856 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156, journal: Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156, size=444 (444bytes) at 1733499656827Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156 so closing down at 1733499656837 (+10 ms)Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000008-testsplit%2C16010%2C1.1733499656156.temp at 1733499656842 (+5 ms)3 split writer threads finished at 1733499656842Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000008-testsplit%2C16010%2C1.1733499656156.temp (wrote 3 edits, skipped 0 edits in 0 ms) at 1733499656853 (+11 ms)Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000008-testsplit%2C16010%2C1.1733499656156.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000010 at 1733499656855 (+2 ms)Processed 3 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156, size=444, length=444, corrupted=false, cancelled=false at 1733499656856 (+1 ms) 2024-12-06T15:40:56,859 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656156 to hdfs://localhost:37105/user/jenkins/test-data/8b6337b1-183f-37ae-d1f3-fef1d3c371dc/oldWALs/testsplit%2C16010%2C1.1733499656156 2024-12-06T15:40:56,860 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193, size=444 (444bytes) 2024-12-06T15:40:56,860 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193 2024-12-06T15:40:56,861 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193 after 1ms 2024-12-06T15:40:56,865 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193 took 6ms 2024-12-06T15:40:56,867 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193 so closing down 2024-12-06T15:40:56,867 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T15:40:56,870 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000011-testsplit%2C16010%2C1.1733499656193.temp 2024-12-06T15:40:56,872 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000011-testsplit%2C16010%2C1.1733499656193.temp 2024-12-06T15:40:56,872 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T15:40:56,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741840_1016 (size=444) 2024-12-06T15:40:56,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741840_1016 (size=444) 2024-12-06T15:40:56,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741840_1016 (size=444) 2024-12-06T15:40:56,884 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000011-testsplit%2C16010%2C1.1733499656193.temp (wrote 3 edits, skipped 0 edits in 0 ms) 2024-12-06T15:40:56,887 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000011-testsplit%2C16010%2C1.1733499656193.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000013 2024-12-06T15:40:56,887 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193, size=444, length=444, corrupted=false, cancelled=false 2024-12-06T15:40:56,887 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193, journal: Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193, size=444 (444bytes) at 1733499656860Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193 so closing down at 1733499656867 (+7 ms)Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000011-testsplit%2C16010%2C1.1733499656193.temp at 1733499656872 (+5 ms)3 split writer threads finished at 1733499656873 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000011-testsplit%2C16010%2C1.1733499656193.temp (wrote 3 edits, skipped 0 edits in 0 ms) at 1733499656884 (+11 ms)Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000011-testsplit%2C16010%2C1.1733499656193.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000013 at 1733499656887 (+3 ms)Processed 3 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193, size=444, length=444, corrupted=false, cancelled=false at 1733499656887 2024-12-06T15:40:56,891 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656193 to hdfs://localhost:37105/user/jenkins/test-data/8b6337b1-183f-37ae-d1f3-fef1d3c371dc/oldWALs/testsplit%2C16010%2C1.1733499656193 2024-12-06T15:40:56,891 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259, size=444 (444bytes) 2024-12-06T15:40:56,892 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259 2024-12-06T15:40:56,892 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259 after 0ms 2024-12-06T15:40:56,897 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259 took 6ms 2024-12-06T15:40:56,899 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259 so closing down 2024-12-06T15:40:56,899 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T15:40:56,902 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000014-testsplit%2C16010%2C1.1733499656259.temp 2024-12-06T15:40:56,904 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000014-testsplit%2C16010%2C1.1733499656259.temp 2024-12-06T15:40:56,904 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T15:40:56,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741841_1017 (size=444) 2024-12-06T15:40:56,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741841_1017 (size=444) 2024-12-06T15:40:56,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741841_1017 (size=444) 2024-12-06T15:40:56,917 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000014-testsplit%2C16010%2C1.1733499656259.temp (wrote 3 edits, skipped 0 edits in 0 ms) 2024-12-06T15:40:56,919 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000014-testsplit%2C16010%2C1.1733499656259.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000016 2024-12-06T15:40:56,919 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259, size=444, length=444, corrupted=false, cancelled=false 2024-12-06T15:40:56,920 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259, journal: Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259, size=444 (444bytes) at 1733499656892Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259 so closing down at 1733499656899 (+7 ms)Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000014-testsplit%2C16010%2C1.1733499656259.temp at 1733499656904 (+5 ms)3 split writer threads finished at 1733499656904Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000014-testsplit%2C16010%2C1.1733499656259.temp (wrote 3 edits, skipped 0 edits in 0 ms) at 1733499656917 (+13 ms)Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000014-testsplit%2C16010%2C1.1733499656259.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000016 at 1733499656919 (+2 ms)Processed 3 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259, size=444, length=444, corrupted=false, cancelled=false at 1733499656919 2024-12-06T15:40:56,923 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656259 to hdfs://localhost:37105/user/jenkins/test-data/8b6337b1-183f-37ae-d1f3-fef1d3c371dc/oldWALs/testsplit%2C16010%2C1.1733499656259 2024-12-06T15:40:56,923 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297, size=444 (444bytes) 2024-12-06T15:40:56,923 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 2024-12-06T15:40:56,924 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 after 1ms 2024-12-06T15:40:56,928 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 took 5ms 2024-12-06T15:40:56,930 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 so closing down 2024-12-06T15:40:56,930 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T15:40:56,933 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000017-testsplit%2C16010%2C1.1733499656297.temp 2024-12-06T15:40:56,935 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000017-testsplit%2C16010%2C1.1733499656297.temp 2024-12-06T15:40:56,936 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T15:40:56,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741842_1018 (size=444) 2024-12-06T15:40:56,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741842_1018 (size=444) 2024-12-06T15:40:56,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741842_1018 (size=444) 2024-12-06T15:40:56,946 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000017-testsplit%2C16010%2C1.1733499656297.temp (wrote 3 edits, skipped 0 edits in 0 ms) 2024-12-06T15:40:56,949 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000017-testsplit%2C16010%2C1.1733499656297.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000019 2024-12-06T15:40:56,949 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3 edits across 1 Regions in 20 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297, size=444, length=444, corrupted=false, cancelled=false 2024-12-06T15:40:56,949 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297, journal: Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297, size=444 (444bytes) at 1733499656923Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 so closing down at 1733499656930 (+7 ms)Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000017-testsplit%2C16010%2C1.1733499656297.temp at 1733499656935 (+5 ms)3 split writer threads finished at 1733499656936 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000017-testsplit%2C16010%2C1.1733499656297.temp (wrote 3 edits, skipped 0 edits in 0 ms) at 1733499656946 (+10 ms)Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000017-testsplit%2C16010%2C1.1733499656297.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000019 at 1733499656949 (+3 ms)Processed 3 edits across 1 Regions in 20 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297, size=444, length=444, corrupted=false, cancelled=false at 1733499656949 2024-12-06T15:40:56,953 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656297 to hdfs://localhost:37105/user/jenkins/test-data/8b6337b1-183f-37ae-d1f3-fef1d3c371dc/oldWALs/testsplit%2C16010%2C1.1733499656297 2024-12-06T15:40:56,953 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347, size=444 (444bytes) 2024-12-06T15:40:56,953 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347 2024-12-06T15:40:56,954 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347 after 1ms 2024-12-06T15:40:56,958 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347 took 5ms 2024-12-06T15:40:56,960 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347 so closing down 2024-12-06T15:40:56,960 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T15:40:56,963 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000020-testsplit%2C16010%2C1.1733499656347.temp 2024-12-06T15:40:56,964 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000020-testsplit%2C16010%2C1.1733499656347.temp 2024-12-06T15:40:56,965 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T15:40:56,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741843_1019 (size=444) 2024-12-06T15:40:56,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741843_1019 (size=444) 2024-12-06T15:40:56,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741843_1019 (size=444) 2024-12-06T15:40:56,975 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000020-testsplit%2C16010%2C1.1733499656347.temp (wrote 3 edits, skipped 0 edits in 0 ms) 2024-12-06T15:40:56,977 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000020-testsplit%2C16010%2C1.1733499656347.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000022 2024-12-06T15:40:56,978 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3 edits across 1 Regions in 20 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347, size=444, length=444, corrupted=false, cancelled=false 2024-12-06T15:40:56,978 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347, journal: Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347, size=444 (444bytes) at 1733499656953Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347 so closing down at 1733499656960 (+7 ms)Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000020-testsplit%2C16010%2C1.1733499656347.temp at 1733499656965 (+5 ms)3 split writer threads finished at 1733499656965Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000020-testsplit%2C16010%2C1.1733499656347.temp (wrote 3 edits, skipped 0 edits in 0 ms) at 1733499656975 (+10 ms)Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000020-testsplit%2C16010%2C1.1733499656347.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000022 at 1733499656978 (+3 ms)Processed 3 edits across 1 Regions in 20 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347, size=444, length=444, corrupted=false, cancelled=false at 1733499656978 2024-12-06T15:40:56,981 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656347 to hdfs://localhost:37105/user/jenkins/test-data/8b6337b1-183f-37ae-d1f3-fef1d3c371dc/oldWALs/testsplit%2C16010%2C1.1733499656347 2024-12-06T15:40:56,981 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390, size=444 (444bytes) 2024-12-06T15:40:56,982 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390 2024-12-06T15:40:56,982 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390 after 0ms 2024-12-06T15:40:56,986 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390 took 5ms 2024-12-06T15:40:56,988 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390 so closing down 2024-12-06T15:40:56,988 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T15:40:56,991 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000023-testsplit%2C16010%2C1.1733499656390.temp 2024-12-06T15:40:56,992 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000023-testsplit%2C16010%2C1.1733499656390.temp 2024-12-06T15:40:56,993 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T15:40:57,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741844_1020 (size=444) 2024-12-06T15:40:57,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741844_1020 (size=444) 2024-12-06T15:40:57,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741844_1020 (size=444) 2024-12-06T15:40:57,005 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000023-testsplit%2C16010%2C1.1733499656390.temp (wrote 3 edits, skipped 0 edits in 0 ms) 2024-12-06T15:40:57,007 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000023-testsplit%2C16010%2C1.1733499656390.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000025 2024-12-06T15:40:57,007 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3 edits across 1 Regions in 20 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390, size=444, length=444, corrupted=false, cancelled=false 2024-12-06T15:40:57,007 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390, journal: Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390, size=444 (444bytes) at 1733499656982Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390 so closing down at 1733499656988 (+6 ms)Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000023-testsplit%2C16010%2C1.1733499656390.temp at 1733499656992 (+4 ms)3 split writer threads finished at 1733499656993 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000023-testsplit%2C16010%2C1.1733499656390.temp (wrote 3 edits, skipped 0 edits in 0 ms) at 1733499657005 (+12 ms)Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000023-testsplit%2C16010%2C1.1733499656390.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000025 at 1733499657007 (+2 ms)Processed 3 edits across 1 Regions in 20 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390, size=444, length=444, corrupted=false, cancelled=false at 1733499657007 2024-12-06T15:40:57,010 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656390 to hdfs://localhost:37105/user/jenkins/test-data/8b6337b1-183f-37ae-d1f3-fef1d3c371dc/oldWALs/testsplit%2C16010%2C1.1733499656390 2024-12-06T15:40:57,010 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430, size=444 (444bytes) 2024-12-06T15:40:57,010 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430 2024-12-06T15:40:57,011 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430 after 0ms 2024-12-06T15:40:57,014 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430 took 4ms 2024-12-06T15:40:57,016 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430 so closing down 2024-12-06T15:40:57,017 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T15:40:57,019 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000026-testsplit%2C16010%2C1.1733499656430.temp 2024-12-06T15:40:57,020 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000026-testsplit%2C16010%2C1.1733499656430.temp 2024-12-06T15:40:57,021 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T15:40:57,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741845_1021 (size=444) 2024-12-06T15:40:57,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741845_1021 (size=444) 2024-12-06T15:40:57,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741845_1021 (size=444) 2024-12-06T15:40:57,030 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000026-testsplit%2C16010%2C1.1733499656430.temp (wrote 3 edits, skipped 0 edits in 0 ms) 2024-12-06T15:40:57,032 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000026-testsplit%2C16010%2C1.1733499656430.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000028 2024-12-06T15:40:57,033 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430, size=444, length=444, corrupted=false, cancelled=false 2024-12-06T15:40:57,033 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430, journal: Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430, size=444 (444bytes) at 1733499657010Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430 so closing down at 1733499657016 (+6 ms)Creating recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000026-testsplit%2C16010%2C1.1733499656430.temp at 1733499657020 (+4 ms)3 split writer threads finished at 1733499657021 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000026-testsplit%2C16010%2C1.1733499656430.temp (wrote 3 edits, skipped 0 edits in 0 ms) at 1733499657030 (+9 ms)Rename recovered edits hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000026-testsplit%2C16010%2C1.1733499656430.temp to hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000028 at 1733499657032 (+2 ms)Processed 3 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430, size=444, length=444, corrupted=false, cancelled=false at 1733499657033 (+1 ms) 2024-12-06T15:40:57,037 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656430 to hdfs://localhost:37105/user/jenkins/test-data/8b6337b1-183f-37ae-d1f3-fef1d3c371dc/oldWALs/testsplit%2C16010%2C1.1733499656430 2024-12-06T15:40:57,037 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656475, size=93 (93bytes) 2024-12-06T15:40:57,037 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656475 2024-12-06T15:40:57,038 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656475 after 1ms 2024-12-06T15:40:57,043 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656475 took 6ms 2024-12-06T15:40:57,045 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656475 so closing down 2024-12-06T15:40:57,045 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T15:40:57,045 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T15:40:57,045 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 0 edits across 0 Regions in 0 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656475, size=93, length=93, corrupted=false, cancelled=false 2024-12-06T15:40:57,045 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656475, journal: Splitting hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656475, size=93 (93bytes) at 1733499657037Finishing writing output for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656475 so closing down at 1733499657045 (+8 ms)3 split writer threads finished at 1733499657045Processed 0 edits across 0 Regions in 0 ms; skipped=0; WAL=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656475, size=93, length=93, corrupted=false, cancelled=false at 1733499657045 2024-12-06T15:40:57,050 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testsplit,16010,1/testsplit%2C16010%2C1.1733499656475 to hdfs://localhost:37105/user/jenkins/test-data/8b6337b1-183f-37ae-d1f3-fef1d3c371dc/oldWALs/testsplit%2C16010%2C1.1733499656475 2024-12-06T15:40:57,051 INFO [Time-limited test {}] wal.TestWALFactory(322): Verifying=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000004 2024-12-06T15:40:57,058 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=-1, newseqno=2 2024-12-06T15:40:57,059 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=2, newseqno=3 2024-12-06T15:40:57,059 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=3, newseqno=4 2024-12-06T15:40:57,059 INFO [Time-limited test {}] wal.TestWALFactory(322): Verifying=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000007 2024-12-06T15:40:57,065 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=-1, newseqno=5 2024-12-06T15:40:57,066 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=5, newseqno=6 2024-12-06T15:40:57,066 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=6, newseqno=7 2024-12-06T15:40:57,066 INFO [Time-limited test {}] wal.TestWALFactory(322): Verifying=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000010 2024-12-06T15:40:57,072 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=-1, newseqno=8 2024-12-06T15:40:57,072 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=8, newseqno=9 2024-12-06T15:40:57,073 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=9, newseqno=10 2024-12-06T15:40:57,073 INFO [Time-limited test {}] wal.TestWALFactory(322): Verifying=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000013 2024-12-06T15:40:57,078 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=-1, newseqno=11 2024-12-06T15:40:57,079 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=11, newseqno=12 2024-12-06T15:40:57,079 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=12, newseqno=13 2024-12-06T15:40:57,079 INFO [Time-limited test {}] wal.TestWALFactory(322): Verifying=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000016 2024-12-06T15:40:57,084 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=-1, newseqno=14 2024-12-06T15:40:57,085 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=14, newseqno=15 2024-12-06T15:40:57,085 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=15, newseqno=16 2024-12-06T15:40:57,085 INFO [Time-limited test {}] wal.TestWALFactory(322): Verifying=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000019 2024-12-06T15:40:57,090 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=-1, newseqno=17 2024-12-06T15:40:57,091 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=17, newseqno=18 2024-12-06T15:40:57,091 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=18, newseqno=19 2024-12-06T15:40:57,091 INFO [Time-limited test {}] wal.TestWALFactory(322): Verifying=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/a362d332728839d676e81c1eb9b5d4bd/recovered.edits/0000000000000000022 2024-12-06T15:40:57,097 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=-1, newseqno=20 2024-12-06T15:40:57,097 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=20, newseqno=21 2024-12-06T15:40:57,097 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=21, newseqno=22 2024-12-06T15:40:57,097 INFO [Time-limited test {}] wal.TestWALFactory(322): Verifying=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/1f2a76a4bc600422aeecea3dcf0fb0b7/recovered.edits/0000000000000000025 2024-12-06T15:40:57,104 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=-1, newseqno=23 2024-12-06T15:40:57,104 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=23, newseqno=24 2024-12-06T15:40:57,104 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=24, newseqno=25 2024-12-06T15:40:57,104 INFO [Time-limited test {}] wal.TestWALFactory(322): Verifying=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/data/default/testSplit/d55d743581859e6a513c51db6249917a/recovered.edits/0000000000000000028 2024-12-06T15:40:57,111 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=-1, newseqno=26 2024-12-06T15:40:57,111 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=26, newseqno=27 2024-12-06T15:40:57,111 INFO [Time-limited test {}] wal.TestWALFactory(335): oldseqno=27, newseqno=28 2024-12-06T15:40:57,116 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL testsplit%2C16010%2C1:(num 1733499656475) 2024-12-06T15:40:57,134 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testSplit Thread=179 (was 164) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1386572978_22 at /127.0.0.1:37040 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1386572978_22 at /127.0.0.1:47258 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-3-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@207968f1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-3-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-3-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-3-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-3-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-3-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-3-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-3-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1386572978_22 at /127.0.0.1:54516 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=477 (was 433) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 418) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10433 (was 10487) 2024-12-06T15:40:57,149 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#canCloseSingleton Thread=179, OpenFileDescriptor=477, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=10431 2024-12-06T15:40:57,150 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,189 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#canCloseSingleton Thread=179 (was 179), OpenFileDescriptor=509 (was 477) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=10431 (was 10431) 2024-12-06T15:40:57,197 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testCustomMetaProvider Thread=179, OpenFileDescriptor=509, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=10431 2024-12-06T15:40:57,197 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,211 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,213 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.IOTestProvider 2024-12-06T15:40:57,222 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testCustomMetaProvider Thread=179 (was 179), OpenFileDescriptor=573 (was 509) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=10429 (was 10431) 2024-12-06T15:40:57,229 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testDifferentWALs Thread=179, OpenFileDescriptor=573, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=10429 2024-12-06T15:40:57,230 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,231 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T15:40:57,236 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-12-06T15:40:57,236 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=testdifferentwals%2C16010%2C1, suffix=, logDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testdifferentwals,16010,1, archiveDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs, maxLogs=1760 2024-12-06T15:40:57,252 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testdifferentwals,16010,1/testdifferentwals%2C16010%2C1.1733499657238, exclude list is [], retry=0 2024-12-06T15:40:57,256 DEBUG [AsyncFSWAL-7-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:57,256 DEBUG [AsyncFSWAL-7-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:57,257 DEBUG [AsyncFSWAL-7-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:57,260 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testdifferentwals,16010,1/testdifferentwals%2C16010%2C1.1733499657238 2024-12-06T15:40:57,260 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36457:36457),(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:41695:41695)] 2024-12-06T15:40:57,260 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,261 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T15:40:57,264 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-12-06T15:40:57,264 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=testdifferentwals%2C16010%2C1.meta, suffix=.meta, logDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testdifferentwals,16010,1, archiveDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs, maxLogs=1760 2024-12-06T15:40:57,279 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testdifferentwals,16010,1/testdifferentwals%2C16010%2C1.meta.1733499657266.meta, exclude list is [], retry=0 2024-12-06T15:40:57,283 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:57,284 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:57,284 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:57,287 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testdifferentwals,16010,1/testdifferentwals%2C16010%2C1.meta.1733499657266.meta 2024-12-06T15:40:57,288 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41695:41695),(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:36457:36457)] 2024-12-06T15:40:57,288 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,289 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor rep 2024-12-06T15:40:57,292 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-12-06T15:40:57,293 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=testdifferentwals%2C16010%2C1.rep, suffix=, logDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testdifferentwals,16010,1, archiveDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs, maxLogs=1760 2024-12-06T15:40:57,308 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testdifferentwals,16010,1/testdifferentwals%2C16010%2C1.rep.1733499657294, exclude list is [], retry=0 2024-12-06T15:40:57,312 DEBUG [AsyncFSWAL-9-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:57,312 DEBUG [AsyncFSWAL-9-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:57,312 DEBUG [AsyncFSWAL-9-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:57,316 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testdifferentwals,16010,1/testdifferentwals%2C16010%2C1.rep.1733499657294 2024-12-06T15:40:57,316 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:41695:41695),(127.0.0.1/127.0.0.1:36457:36457)] 2024-12-06T15:40:57,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741847_1023 (size=93) 2024-12-06T15:40:57,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741847_1023 (size=93) 2024-12-06T15:40:57,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741847_1023 (size=93) 2024-12-06T15:40:57,325 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs 2024-12-06T15:40:57,325 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL testdifferentwals%2C16010%2C1.meta:.meta(num 1733499657266) 2024-12-06T15:40:57,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741848_1024 (size=93) 2024-12-06T15:40:57,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741848_1024 (size=93) 2024-12-06T15:40:57,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741848_1024 (size=93) 2024-12-06T15:40:57,333 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs 2024-12-06T15:40:57,333 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL testdifferentwals%2C16010%2C1.rep:(num 1733499657294) 2024-12-06T15:40:57,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741846_1022 (size=93) 2024-12-06T15:40:57,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741846_1022 (size=93) 2024-12-06T15:40:57,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741846_1022 (size=93) 2024-12-06T15:40:57,341 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs 2024-12-06T15:40:57,341 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL testdifferentwals%2C16010%2C1:(num 1733499657238) 2024-12-06T15:40:57,353 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testDifferentWALs Thread=188 (was 179) Potentially hanging thread: AsyncFSWAL-9-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-9-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-7-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-9-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-7-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-7-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=675 (was 573) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=10420 (was 10429) 2024-12-06T15:40:57,363 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testCustomProvider Thread=188, OpenFileDescriptor=675, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=10419 2024-12-06T15:40:57,363 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,377 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.IOTestProvider 2024-12-06T15:40:57,377 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.IOTestProvider 2024-12-06T15:40:57,387 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testCustomProvider Thread=188 (was 188), OpenFileDescriptor=707 (was 675) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=10416 (was 10419) 2024-12-06T15:40:57,397 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#Broken_testSync Thread=188, OpenFileDescriptor=707, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=10416 2024-12-06T15:40:57,397 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741849_1025 (size=15) 2024-12-06T15:40:57,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741849_1025 (size=15) 2024-12-06T15:40:57,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741849_1025 (size=15) 2024-12-06T15:40:57,479 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T15:40:57,483 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-12-06T15:40:57,484 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=broken_testsync%2C16010%2C1, suffix=, logDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/broken_testsync,16010,1, archiveDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs, maxLogs=1760 2024-12-06T15:40:57,505 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/broken_testsync,16010,1/broken_testsync%2C16010%2C1.1733499657485, exclude list is [], retry=0 2024-12-06T15:40:57,509 DEBUG [AsyncFSWAL-11-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:57,509 DEBUG [AsyncFSWAL-11-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:57,509 DEBUG [AsyncFSWAL-11-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:57,514 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/broken_testsync,16010,1/broken_testsync%2C16010%2C1.1733499657485 2024-12-06T15:40:57,515 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41695:41695),(127.0.0.1/127.0.0.1:36457:36457),(127.0.0.1/127.0.0.1:37899:37899)] 2024-12-06T15:40:57,530 WARN [Time-limited test {}] wal.NoEOFWALStreamReader(50): Got EOF while reading java.io.EOFException: EOF while reading message size at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.parseDelimitedFrom(ProtobufUtil.java:3839) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALStreamReader.next(ProtobufWALStreamReader.java:56) ~[classes/:?] at org.apache.hadoop.hbase.wal.NoEOFWALStreamReader.next(NoEOFWALStreamReader.java:48) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.NoEOFWALStreamReader.count(NoEOFWALStreamReader.java:68) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.NoEOFWALStreamReader.count(NoEOFWALStreamReader.java:92) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.TestWALFactory.Broken_testSync(TestWALFactory.java:283) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:40:57,544 WARN [Time-limited test {}] wal.NoEOFWALStreamReader(50): Got EOF while reading java.io.EOFException: EOF while reading message size at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.parseDelimitedFrom(ProtobufUtil.java:3839) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALStreamReader.next(ProtobufWALStreamReader.java:56) ~[classes/:?] at org.apache.hadoop.hbase.wal.NoEOFWALStreamReader.next(NoEOFWALStreamReader.java:48) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.NoEOFWALStreamReader.count(NoEOFWALStreamReader.java:68) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.NoEOFWALStreamReader.count(NoEOFWALStreamReader.java:92) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.TestWALFactory.Broken_testSync(TestWALFactory.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:40:57,554 WARN [Time-limited test {}] wal.NoEOFWALStreamReader(50): Got EOF while reading java.io.EOFException: EOF while reading message size at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.parseDelimitedFrom(ProtobufUtil.java:3839) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALStreamReader.next(ProtobufWALStreamReader.java:56) ~[classes/:?] at org.apache.hadoop.hbase.wal.NoEOFWALStreamReader.next(NoEOFWALStreamReader.java:48) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.NoEOFWALStreamReader.count(NoEOFWALStreamReader.java:68) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.NoEOFWALStreamReader.count(NoEOFWALStreamReader.java:92) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.TestWALFactory.Broken_testSync(TestWALFactory.java:298) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:40:57,588 DEBUG [AsyncFSWAL-0-hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886-prefix:broken_testsync,16010,1 {}] wal.AbstractFSWAL(1598): Requesting log roll because of file size threshold; length=10503035, logrollsize=1048576 2024-12-06T15:40:57,683 WARN [Time-limited test {}] wal.NoEOFWALStreamReader(50): Got EOF while reading java.io.EOFException: EOF while reading message size at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.parseDelimitedFrom(ProtobufUtil.java:3839) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALStreamReader.next(ProtobufWALStreamReader.java:56) ~[classes/:?] at org.apache.hadoop.hbase.wal.NoEOFWALStreamReader.next(NoEOFWALStreamReader.java:48) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.NoEOFWALStreamReader.count(NoEOFWALStreamReader.java:68) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.NoEOFWALStreamReader.count(NoEOFWALStreamReader.java:92) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.TestWALFactory.Broken_testSync(TestWALFactory.java:311) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:40:57,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741850_1026 (size=21000313) 2024-12-06T15:40:57,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741850_1026 (size=21000313) 2024-12-06T15:40:57,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741850_1026 (size=21000313) 2024-12-06T15:40:57,740 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs 2024-12-06T15:40:57,740 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL broken_testsync%2C16010%2C1:(num 1733499657485) 2024-12-06T15:40:57,751 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#Broken_testSync Thread=191 (was 188) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1386572978_22 at /127.0.0.1:37040 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1386572978_22 at /127.0.0.1:47258 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1386572978_22 at /127.0.0.1:54664 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-11-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-11-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-11-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=739 (was 707) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=10254 (was 10416) 2024-12-06T15:40:57,759 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testCustomReplicationProvider Thread=191, OpenFileDescriptor=739, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=10253 2024-12-06T15:40:57,760 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,774 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,775 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.IOTestProvider 2024-12-06T15:40:57,785 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testCustomReplicationProvider Thread=191 (was 191), OpenFileDescriptor=803 (was 739) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=10251 (was 10253) 2024-12-06T15:40:57,793 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testOnlySetMetaWALProvider Thread=191, OpenFileDescriptor=803, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=10250 2024-12-06T15:40:57,793 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,807 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,808 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,817 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testOnlySetMetaWALProvider Thread=191 (was 191), OpenFileDescriptor=899 (was 803) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=10247 (was 10250) 2024-12-06T15:40:57,825 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testDefaultProvider Thread=191, OpenFileDescriptor=899, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=10246 2024-12-06T15:40:57,826 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,840 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,841 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,852 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testDefaultProvider Thread=191 (was 191), OpenFileDescriptor=995 (was 899) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=10247 (was 10246) - AvailableMemoryMB LEAK? - 2024-12-06T15:40:57,861 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testReaderClosedOnBadCodec Thread=191, OpenFileDescriptor=995, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=10246 2024-12-06T15:40:57,861 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,889 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,899 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T15:40:57,903 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-12-06T15:40:57,904 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=testreaderclosedonbadcodec%2C16010%2C1, suffix=, logDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testreaderclosedonbadcodec,16010,1, archiveDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs, maxLogs=1760 2024-12-06T15:40:57,921 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testreaderclosedonbadcodec,16010,1/testreaderclosedonbadcodec%2C16010%2C1.1733499657905, exclude list is [], retry=0 2024-12-06T15:40:57,925 DEBUG [AsyncFSWAL-21-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:57,925 DEBUG [AsyncFSWAL-21-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:57,926 DEBUG [AsyncFSWAL-21-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:57,929 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testreaderclosedonbadcodec,16010,1/testreaderclosedonbadcodec%2C16010%2C1.1733499657905 2024-12-06T15:40:57,929 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:36457:36457),(127.0.0.1/127.0.0.1:41695:41695)] 2024-12-06T15:40:57,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741851_1027 (size=222) 2024-12-06T15:40:57,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741851_1027 (size=222) 2024-12-06T15:40:57,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741851_1027 (size=222) 2024-12-06T15:40:57,958 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testReaderClosedOnBadCodec Thread=194 (was 191) Potentially hanging thread: AsyncFSWAL-21-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-21-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1386572978_22 at /127.0.0.1:54664 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-21-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1059 (was 995) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=10243 (was 10246) 2024-12-06T15:40:57,961 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1059 is superior to 1024 2024-12-06T15:40:57,970 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testAppend Thread=194, OpenFileDescriptor=1059, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=10242 2024-12-06T15:40:57,970 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1059 is superior to 1024 2024-12-06T15:40:57,970 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:57,972 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T15:40:57,976 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-12-06T15:40:57,977 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=testappend%2C16010%2C1, suffix=, logDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappend,16010,1, archiveDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs, maxLogs=1760 2024-12-06T15:40:57,999 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappend,16010,1/testappend%2C16010%2C1.1733499657978, exclude list is [], retry=0 2024-12-06T15:40:58,003 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:58,004 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:58,004 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:58,008 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappend,16010,1/testappend%2C16010%2C1.1733499657978 2024-12-06T15:40:58,009 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36457:36457),(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:41695:41695)] 2024-12-06T15:40:58,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741852_1028 (size=513) 2024-12-06T15:40:58,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741852_1028 (size=513) 2024-12-06T15:40:58,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741852_1028 (size=513) testAppend/fefae544be332c7cbe66208feca01e59/2 row/column:0/1733499657971/Put/vlen=1/seqid=0 testAppend/fefae544be332c7cbe66208feca01e59/2 row/column:1/1733499657971/Put/vlen=1/seqid=0 testAppend/fefae544be332c7cbe66208feca01e59/2 row/column:2/1733499657971/Put/vlen=1/seqid=0 testAppend/fefae544be332c7cbe66208feca01e59/2 row/column:3/1733499657971/Put/vlen=1/seqid=0 testAppend/fefae544be332c7cbe66208feca01e59/2 row/column:4/1733499657971/Put/vlen=1/seqid=0 testAppend/fefae544be332c7cbe66208feca01e59/2 row/column:5/1733499657971/Put/vlen=1/seqid=0 testAppend/fefae544be332c7cbe66208feca01e59/2 row/column:6/1733499657971/Put/vlen=1/seqid=0 testAppend/fefae544be332c7cbe66208feca01e59/2 row/column:7/1733499657971/Put/vlen=1/seqid=0 testAppend/fefae544be332c7cbe66208feca01e59/2 row/column:8/1733499657971/Put/vlen=1/seqid=0 testAppend/fefae544be332c7cbe66208feca01e59/2 row/column:9/1733499657971/Put/vlen=1/seqid=0 2024-12-06T15:40:58,026 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs 2024-12-06T15:40:58,026 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL testappend%2C16010%2C1:(num 1733499657978) 2024-12-06T15:40:58,039 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testAppend Thread=197 (was 194) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1386572978_22 at /127.0.0.1:37040 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1091 (was 1059) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=10241 (was 10242) 2024-12-06T15:40:58,039 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1091 is superior to 1024 2024-12-06T15:40:58,050 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testWALCoprocessorLoaded Thread=197, OpenFileDescriptor=1091, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=10240 2024-12-06T15:40:58,050 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1091 is superior to 1024 2024-12-06T15:40:58,050 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:58,051 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T15:40:58,057 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-12-06T15:40:58,057 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=testwalcoprocessorloaded%2C16010%2C1, suffix=, logDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testwalcoprocessorloaded,16010,1, archiveDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs, maxLogs=1760 2024-12-06T15:40:58,079 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testwalcoprocessorloaded,16010,1/testwalcoprocessorloaded%2C16010%2C1.1733499658058, exclude list is [], retry=0 2024-12-06T15:40:58,083 DEBUG [AsyncFSWAL-23-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:58,084 DEBUG [AsyncFSWAL-23-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:58,085 DEBUG [AsyncFSWAL-23-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:58,091 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testwalcoprocessorloaded,16010,1/testwalcoprocessorloaded%2C16010%2C1.1733499658058 2024-12-06T15:40:58,092 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36457:36457),(127.0.0.1/127.0.0.1:41695:41695),(127.0.0.1/127.0.0.1:37899:37899)] 2024-12-06T15:40:58,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741853_1029 (size=93) 2024-12-06T15:40:58,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741853_1029 (size=93) 2024-12-06T15:40:58,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741853_1029 (size=93) 2024-12-06T15:40:58,110 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs 2024-12-06T15:40:58,110 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL testwalcoprocessorloaded%2C16010%2C1:(num 1733499658058) 2024-12-06T15:40:58,124 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testWALCoprocessorLoaded Thread=200 (was 197) Potentially hanging thread: AsyncFSWAL-23-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-23-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-23-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1123 (was 1091) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=10234 (was 10240) 2024-12-06T15:40:58,124 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1123 is superior to 1024 2024-12-06T15:40:58,133 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testOnlySetWALProvider Thread=200, OpenFileDescriptor=1123, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=10233 2024-12-06T15:40:58,134 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1123 is superior to 1024 2024-12-06T15:40:58,134 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:58,154 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-06T15:40:58,156 INFO [Time-limited test {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-06T15:40:58,157 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-06T15:40:58,157 INFO [Time-limited test {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-06T15:40:58,169 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testOnlySetWALProvider Thread=200 (was 200), OpenFileDescriptor=1155 (was 1123) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=10230 (was 10233) 2024-12-06T15:40:58,169 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1155 is superior to 1024 2024-12-06T15:40:58,179 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testAppendClose Thread=200, OpenFileDescriptor=1155, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=10229 2024-12-06T15:40:58,179 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1155 is superior to 1024 2024-12-06T15:40:58,180 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:40:58,181 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T15:40:58,186 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-12-06T15:40:58,187 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=testappendclose%2C16010%2C1, suffix=, logDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1, archiveDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs, maxLogs=1760 2024-12-06T15:40:58,209 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187, exclude list is [], retry=0 2024-12-06T15:40:58,215 DEBUG [AsyncFSWAL-25-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36541,DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36,DISK] 2024-12-06T15:40:58,216 DEBUG [AsyncFSWAL-25-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:40:58,217 DEBUG [AsyncFSWAL-25-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46587,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:40:58,224 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 2024-12-06T15:40:58,228 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36457:36457),(127.0.0.1/127.0.0.1:37899:37899),(127.0.0.1/127.0.0.1:41695:41695)] 2024-12-06T15:40:58,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46587 is added to blk_1073741854_1030 (size=2925) 2024-12-06T15:40:58,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36541 is added to blk_1073741854_1030 (size=2925) 2024-12-06T15:40:58,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741854_1030 (size=2925) 2024-12-06T15:40:58,246 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 with renewLeaseKey: DEFAULT_16492 org.apache.hadoop.hdfs.server.namenode.SafeModeException: Cannot complete file /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187. Name node is in safe mode. It was turned on manually. Use "hdfs dfsadmin -safemode leave" to turn safe mode off. NamenodeHostName:localhost at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.newSafemodeException(FSNamesystem.java:1661) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkNameNodeSafeMode(FSNamesystem.java:1648) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3231) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.wal.TestWALFactory.testAppendClose(TestWALFactory.java:378) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Cannot complete file /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187. Name node is in safe mode. It was turned on manually. Use "hdfs dfsadmin -safemode leave" to turn safe mode off. NamenodeHostName:localhost at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.newSafemodeException(FSNamesystem.java:1661) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkNameNodeSafeMode(FSNamesystem.java:1648) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3231) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 37 more 2024-12-06T15:40:58,253 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dcc801f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:40:58,256 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7c3ad891{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T15:40:58,257 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T15:40:58,257 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13eb518c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T15:40:58,257 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef440a4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,STOPPED} 2024-12-06T15:40:58,260 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T15:40:58,260 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T15:40:58,260 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1829792084-172.17.0.2-1733499651885 (Datanode Uuid 0d6e69cc-7877-4434-b2cb-608913e6b424) service to localhost/127.0.0.1:37105 2024-12-06T15:40:58,261 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T15:40:58,262 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data5/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:40:58,262 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data6/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:40:58,263 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T15:40:58,266 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7f751333{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:40:58,266 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@502117d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T15:40:58,266 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T15:40:58,267 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62756a49{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T15:40:58,267 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@997ee7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,STOPPED} 2024-12-06T15:40:58,269 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T15:40:58,269 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T15:40:58,269 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1829792084-172.17.0.2-1733499651885 (Datanode Uuid 466c1d1c-bc74-4274-ab94-ebed5ce6b316) service to localhost/127.0.0.1:37105 2024-12-06T15:40:58,269 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T15:40:58,269 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data3/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:40:58,270 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data4/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:40:58,270 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T15:40:58,279 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28de11c6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:40:58,279 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@52cb6d33{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T15:40:58,279 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T15:40:58,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6034ceb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T15:40:58,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6cfcc17f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,STOPPED} 2024-12-06T15:40:58,281 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T15:40:58,281 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1829792084-172.17.0.2-1733499651885 (Datanode Uuid c262f2ad-b09a-4750-9e9a-43ee971188b0) service to localhost/127.0.0.1:37105 2024-12-06T15:40:58,281 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T15:40:58,281 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T15:40:58,282 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data1/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:40:58,282 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data2/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:40:58,282 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T15:40:58,295 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@179dea64{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T15:40:58,296 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7975eec2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T15:40:58,296 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T15:40:58,296 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d3e0f30{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T15:40:58,296 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70940934{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,STOPPED} 2024-12-06T15:40:58,304 WARN [Close-WAL-Writer-0 {}] wal.AsyncProtobufLogWriter(165): normal close failed, try recover java.lang.NullPointerException: Cannot invoke "org.apache.hbase.thirdparty.io.netty.buffer.ByteBuf.ensureWritable(int)" because "this.buf" is null at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.writeInt(FanOutOneBlockAsyncDFSOutput.java:391) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.lambda$writeWALTrailerAndMagic$3(AsyncProtobufLogWriter.java:247) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALMetadata(AsyncProtobufLogWriter.java:203) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALTrailerAndMagic(AsyncProtobufLogWriter.java:240) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.writeWALTrailer(AbstractProtobufLogWriter.java:252) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:162) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:40:58,305 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 2024-12-06T15:40:58,305 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=0 on file=/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 after 0ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:40:58,305 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.InterruptedIOException: Operation cancelled at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.checkIfCancelled(RecoverLeaseFSUtils.java:269) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:159) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:40:58,305 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 2024-12-06T15:40:58,306 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=0 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 after 1ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:41:02,306 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=1 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 after 4001ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:41:03,304 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.async.wait.on.shutdown.seconds" 2024-12-06T15:41:03,304 INFO [Time-limited test {}] wal.TestWALFactory(387): STOPPED first instance of the cluster 2024-12-06T15:41:03,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/test.cache.data in system properties and HBase conf 2024-12-06T15:41:03,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.tmp.dir in system properties and HBase conf 2024-12-06T15:41:03,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir in system properties and HBase conf 2024-12-06T15:41:03,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-06T15:41:03,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-06T15:41:03,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-06T15:41:03,306 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T15:41:03,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-06T15:41:03,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-06T15:41:03,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-06T15:41:03,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T15:41:03,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-06T15:41:03,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-06T15:41:03,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T15:41:03,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T15:41:03,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-06T15:41:03,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/nfs.dump.dir in system properties and HBase conf 2024-12-06T15:41:03,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/java.io.tmpdir in system properties and HBase conf 2024-12-06T15:41:03,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T15:41:03,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-06T15:41:03,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-06T15:41:03,309 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-12-06T15:41:03,310 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 after 5005ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:41:03,344 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:41:03,361 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T15:41:03,376 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T15:41:03,376 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T15:41:03,377 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T15:41:03,379 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:41:03,383 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39fa668d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,AVAILABLE} 2024-12-06T15:41:03,384 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79f425ce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T15:41:03,624 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@754e9679{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/java.io.tmpdir/jetty-localhost-40627-hadoop-hdfs-3_4_1-tests_jar-_-any-4517343618417155744/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T15:41:03,626 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@412c2abe{HTTP/1.1, (http/1.1)}{localhost:40627} 2024-12-06T15:41:03,626 INFO [Time-limited test {}] server.Server(415): Started @14865ms 2024-12-06T15:41:03,685 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-06T15:41:04,095 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:41:04,111 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T15:41:04,148 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T15:41:04,148 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T15:41:04,148 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T15:41:04,149 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ab9b30d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,AVAILABLE} 2024-12-06T15:41:04,150 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ad6957a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T15:41:04,311 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-12-06T15:41:04,327 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79863ca9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/java.io.tmpdir/jetty-localhost-41577-hadoop-hdfs-3_4_1-tests_jar-_-any-17519110000291575397/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:41:04,329 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@495dc18a{HTTP/1.1, (http/1.1)}{localhost:41577} 2024-12-06T15:41:04,329 INFO [Time-limited test {}] server.Server(415): Started @15568ms 2024-12-06T15:41:04,333 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T15:41:04,389 WARN [Thread-414 {}] datanode.BPServiceActor(266): Problem connecting to server: localhost/127.0.0.1:37105 2024-12-06T15:41:04,428 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:41:04,437 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T15:41:04,449 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T15:41:04,449 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T15:41:04,450 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-06T15:41:04,452 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c7fe737{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,AVAILABLE} 2024-12-06T15:41:04,453 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c99e40{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T15:41:04,625 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@19bae41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/java.io.tmpdir/jetty-localhost-38291-hadoop-hdfs-3_4_1-tests_jar-_-any-12872324127181295152/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:41:04,626 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1148d54{HTTP/1.1, (http/1.1)}{localhost:38291} 2024-12-06T15:41:04,626 INFO [Time-limited test {}] server.Server(415): Started @15864ms 2024-12-06T15:41:04,629 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T15:41:04,761 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:41:04,771 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T15:41:04,793 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T15:41:04,793 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T15:41:04,793 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-06T15:41:04,794 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@383504c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,AVAILABLE} 2024-12-06T15:41:04,795 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2467ba64{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T15:41:04,948 WARN [Thread-434 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T15:41:04,953 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe4899a2b55880787 with lease ID 0x2f6e50124ec06616: Processing first storage report for DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6 from datanode DatanodeRegistration(127.0.0.1:41809, datanodeUuid=466c1d1c-bc74-4274-ab94-ebed5ce6b316, infoPort=33349, infoSecurePort=0, ipcPort=46873, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:41:04,954 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe4899a2b55880787 with lease ID 0x2f6e50124ec06616: from storage DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6 node DatanodeRegistration(127.0.0.1:41809, datanodeUuid=466c1d1c-bc74-4274-ab94-ebed5ce6b316, infoPort=33349, infoSecurePort=0, ipcPort=46873, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 14, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T15:41:04,954 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe4899a2b55880787 with lease ID 0x2f6e50124ec06616: Processing first storage report for DS-67b24bf5-c01f-4c33-a833-8f1c68f972e8 from datanode DatanodeRegistration(127.0.0.1:41809, datanodeUuid=466c1d1c-bc74-4274-ab94-ebed5ce6b316, infoPort=33349, infoSecurePort=0, ipcPort=46873, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:41:04,954 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe4899a2b55880787 with lease ID 0x2f6e50124ec06616: from storage DS-67b24bf5-c01f-4c33-a833-8f1c68f972e8 node DatanodeRegistration(127.0.0.1:41809, datanodeUuid=466c1d1c-bc74-4274-ab94-ebed5ce6b316, infoPort=33349, infoSecurePort=0, ipcPort=46873, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 14, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T15:41:04,986 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2e5024e1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/java.io.tmpdir/jetty-localhost-38819-hadoop-hdfs-3_4_1-tests_jar-_-any-6759037778201551503/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:41:04,992 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@14033e15{HTTP/1.1, (http/1.1)}{localhost:38819} 2024-12-06T15:41:04,992 INFO [Time-limited test {}] server.Server(415): Started @16231ms 2024-12-06T15:41:04,995 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T15:41:05,145 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:41:05,159 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T15:41:05,164 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T15:41:05,164 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T15:41:05,165 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T15:41:05,173 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@281828e9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,AVAILABLE} 2024-12-06T15:41:05,173 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64c6a8c9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T15:41:05,312 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-12-06T15:41:05,313 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=3 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 after 7008ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:41:05,353 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@45b0aaa2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/java.io.tmpdir/jetty-localhost-41959-hadoop-hdfs-3_4_1-tests_jar-_-any-10977735950199149979/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:41:05,354 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2129ddf3{HTTP/1.1, (http/1.1)}{localhost:41959} 2024-12-06T15:41:05,354 WARN [Thread-466 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T15:41:05,354 INFO [Time-limited test {}] server.Server(415): Started @16593ms 2024-12-06T15:41:05,357 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T15:41:05,367 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7bc3c98d47ab429e with lease ID 0x2f6e50124ec06617: Processing first storage report for DS-4f4b3052-a29f-49e8-a992-229f8202a569 from datanode DatanodeRegistration(127.0.0.1:35821, datanodeUuid=0d6e69cc-7877-4434-b2cb-608913e6b424, infoPort=34965, infoSecurePort=0, ipcPort=34567, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:41:05,367 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7bc3c98d47ab429e with lease ID 0x2f6e50124ec06617: from storage DS-4f4b3052-a29f-49e8-a992-229f8202a569 node DatanodeRegistration(127.0.0.1:35821, datanodeUuid=0d6e69cc-7877-4434-b2cb-608913e6b424, infoPort=34965, infoSecurePort=0, ipcPort=34567, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 15, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T15:41:05,367 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7bc3c98d47ab429e with lease ID 0x2f6e50124ec06617: Processing first storage report for DS-08640c80-06d2-4c05-ad8c-67c9d4028b0c from datanode DatanodeRegistration(127.0.0.1:35821, datanodeUuid=0d6e69cc-7877-4434-b2cb-608913e6b424, infoPort=34965, infoSecurePort=0, ipcPort=34567, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:41:05,368 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7bc3c98d47ab429e with lease ID 0x2f6e50124ec06617: from storage DS-08640c80-06d2-4c05-ad8c-67c9d4028b0c node DatanodeRegistration(127.0.0.1:35821, datanodeUuid=0d6e69cc-7877-4434-b2cb-608913e6b424, infoPort=34965, infoSecurePort=0, ipcPort=34567, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 15, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-06T15:41:05,502 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:41:05,518 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T15:41:05,556 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T15:41:05,556 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T15:41:05,556 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T15:41:05,560 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@53b6a0b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,AVAILABLE} 2024-12-06T15:41:05,561 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1fb841e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T15:41:05,587 WARN [Thread-519 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data7/current/BP-1829792084-172.17.0.2-1733499651885/current, will proceed with Du for space computation calculation, 2024-12-06T15:41:05,588 WARN [Thread-520 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data8/current/BP-1829792084-172.17.0.2-1733499651885/current, will proceed with Du for space computation calculation, 2024-12-06T15:41:05,623 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.WAL.CP_org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor 2024-12-06T15:41:05,625 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.WAL.CP_org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor Metrics about HBase WALObservers 2024-12-06T15:41:05,683 WARN [Thread-499 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T15:41:05,697 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7121e9591dd9d2bf with lease ID 0x2f6e50124ec06618: Processing first storage report for DS-6fd08e09-d138-41e7-a107-37dd8889a593 from datanode DatanodeRegistration(127.0.0.1:41267, datanodeUuid=de72905f-7ca5-4e87-92ad-8eb565185846, infoPort=43633, infoSecurePort=0, ipcPort=38153, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:41:05,697 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7121e9591dd9d2bf with lease ID 0x2f6e50124ec06618: from storage DS-6fd08e09-d138-41e7-a107-37dd8889a593 node DatanodeRegistration(127.0.0.1:41267, datanodeUuid=de72905f-7ca5-4e87-92ad-8eb565185846, infoPort=43633, infoSecurePort=0, ipcPort=38153, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-06T15:41:05,697 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7121e9591dd9d2bf with lease ID 0x2f6e50124ec06618: Processing first storage report for DS-17733c07-c182-4271-b917-c30f952b5c1e from datanode DatanodeRegistration(127.0.0.1:41267, datanodeUuid=de72905f-7ca5-4e87-92ad-8eb565185846, infoPort=43633, infoSecurePort=0, ipcPort=38153, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:41:05,697 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7121e9591dd9d2bf with lease ID 0x2f6e50124ec06618: from storage DS-17733c07-c182-4271-b917-c30f952b5c1e node DatanodeRegistration(127.0.0.1:41267, datanodeUuid=de72905f-7ca5-4e87-92ad-8eb565185846, infoPort=43633, infoSecurePort=0, ipcPort=38153, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T15:41:05,749 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@45a7078e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/java.io.tmpdir/jetty-localhost-34435-hadoop-hdfs-3_4_1-tests_jar-_-any-15502852528539288733/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:41:05,750 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3237b321{HTTP/1.1, (http/1.1)}{localhost:34435} 2024-12-06T15:41:05,750 INFO [Time-limited test {}] server.Server(415): Started @16988ms 2024-12-06T15:41:05,753 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T15:41:06,043 WARN [Thread-545 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data9/current/BP-1829792084-172.17.0.2-1733499651885/current, will proceed with Du for space computation calculation, 2024-12-06T15:41:06,045 WARN [Thread-546 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data10/current/BP-1829792084-172.17.0.2-1733499651885/current, will proceed with Du for space computation calculation, 2024-12-06T15:41:06,129 WARN [Thread-534 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T15:41:06,140 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfd7f12e15be6cea1 with lease ID 0x2f6e50124ec06619: Processing first storage report for DS-877d5313-8b78-4ec1-91c4-7731b03c6bc8 from datanode DatanodeRegistration(127.0.0.1:37449, datanodeUuid=08933cec-be98-46d5-86fb-b05c38673056, infoPort=43479, infoSecurePort=0, ipcPort=45919, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:41:06,140 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfd7f12e15be6cea1 with lease ID 0x2f6e50124ec06619: from storage DS-877d5313-8b78-4ec1-91c4-7731b03c6bc8 node DatanodeRegistration(127.0.0.1:37449, datanodeUuid=08933cec-be98-46d5-86fb-b05c38673056, infoPort=43479, infoSecurePort=0, ipcPort=45919, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T15:41:06,140 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfd7f12e15be6cea1 with lease ID 0x2f6e50124ec06619: Processing first storage report for DS-2029136c-2d4a-4dc9-9432-ee54f5375d19 from datanode DatanodeRegistration(127.0.0.1:37449, datanodeUuid=08933cec-be98-46d5-86fb-b05c38673056, infoPort=43479, infoSecurePort=0, ipcPort=45919, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:41:06,140 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfd7f12e15be6cea1 with lease ID 0x2f6e50124ec06619: from storage DS-2029136c-2d4a-4dc9-9432-ee54f5375d19 node DatanodeRegistration(127.0.0.1:37449, datanodeUuid=08933cec-be98-46d5-86fb-b05c38673056, infoPort=43479, infoSecurePort=0, ipcPort=45919, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T15:41:06,314 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-12-06T15:41:07,315 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-12-06T15:41:08,316 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-12-06T15:41:08,317 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=4 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 after 10012ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:41:08,317 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(205): Cannot recoverLease after trying for 10000ms (hbase.lease.recovery.timeout); continuing, but may be DATALOSS!!!; attempt=4 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 after 10012ms 2024-12-06T15:41:09,530 WARN [Thread-414 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T15:41:09,534 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb069152f0d9ee92e with lease ID 0x2f6e50124ec0661a: Processing first storage report for DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36 from datanode DatanodeRegistration(127.0.0.1:34727, datanodeUuid=c262f2ad-b09a-4750-9e9a-43ee971188b0, infoPort=40111, infoSecurePort=0, ipcPort=44581, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:41:09,534 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb069152f0d9ee92e with lease ID 0x2f6e50124ec0661a: from storage DS-5159a387-3dbb-4a3e-88fd-d8486eb17a36 node DatanodeRegistration(127.0.0.1:34727, datanodeUuid=c262f2ad-b09a-4750-9e9a-43ee971188b0, infoPort=40111, infoSecurePort=0, ipcPort=44581, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 15, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T15:41:09,534 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb069152f0d9ee92e with lease ID 0x2f6e50124ec0661a: Processing first storage report for DS-f2cff151-b014-48d5-92f8-12da8f3c2838 from datanode DatanodeRegistration(127.0.0.1:34727, datanodeUuid=c262f2ad-b09a-4750-9e9a-43ee971188b0, infoPort=40111, infoSecurePort=0, ipcPort=44581, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885) 2024-12-06T15:41:09,534 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb069152f0d9ee92e with lease ID 0x2f6e50124ec0661a: from storage DS-f2cff151-b014-48d5-92f8-12da8f3c2838 node DatanodeRegistration(127.0.0.1:34727, datanodeUuid=c262f2ad-b09a-4750-9e9a-43ee971188b0, infoPort=40111, infoSecurePort=0, ipcPort=44581, storageInfo=lv=-57;cid=testClusterID;nsid=1635299332;c=1733499651885), blocks: 15, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T15:41:09,571 INFO [Time-limited test {}] wal.TestWALFactory(407): STARTED second instance. 2024-12-06T15:41:10,581 INFO [Thread-565 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 2024-12-06T15:41:10,589 WARN [IPC Server handler 4 on default port 37105 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 has not been closed. Lease recovery is in progress. RecoveryId = 1031 for block blk_1073741854_1030 2024-12-06T15:41:10,590 INFO [Thread-565 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 after 2ms 2024-12-06T15:41:14,593 INFO [Thread-565 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testappendclose,16010,1/testappendclose%2C16010%2C1.1733499658187 after 4005ms 2024-12-06T15:41:14,614 WARN [Time-limited test {}] wal.NoEOFWALStreamReader(50): Got EOF while reading java.io.EOFException: EOF while reading message size at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.parseDelimitedFrom(ProtobufUtil.java:3839) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALStreamReader.next(ProtobufWALStreamReader.java:56) ~[classes/:?] at org.apache.hadoop.hbase.wal.NoEOFWALStreamReader.next(NoEOFWALStreamReader.java:48) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.TestWALFactory.testAppendClose(TestWALFactory.java:454) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:41:14,615 ERROR [Time-limited test {}] wal.AbstractWALProvider(289): cleanup WAL failed java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:1731) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:1717) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listStatusInternal(DistributedFileSystem.java:1112) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.access$600(DistributedFileSystem.java:148) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem.java:1187) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem.java:1184) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listStatus(DistributedFileSystem.java:1194) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2078) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2122) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.CommonFSUtils.listStatus(CommonFSUtils.java:602) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.getFiles(AbstractFSWAL.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.close(AbstractFSWAL.java:1235) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.close0(AbstractFSWALProvider.java:170) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.cleanup(AbstractWALProvider.java:287) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.close(AbstractWALProvider.java:304) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.close(WALFactory.java:302) ~[classes/:?] at org.apache.hadoop.hbase.wal.TestWALFactory.tearDown(TestWALFactory.java:130) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:41:14,616 WARN [Time-limited test {}] wal.TestWALFactory(132): Encountered exception while closing wal factory. If you have other errors, this may be the cause. Message: java.io.IOException: Failed to close WALFactory 2024-12-06T15:41:14,616 DEBUG [Time-limited test {}] wal.TestWALFactory(134): Exception details for failure to close wal factory. java.io.IOException: Failed to close WALFactory at org.apache.hadoop.hbase.wal.WALFactory.close(WALFactory.java:308) ~[classes/:?] at org.apache.hadoop.hbase.wal.TestWALFactory.tearDown(TestWALFactory.java:130) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Suppressed: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:1731) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:1717) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listStatusInternal(DistributedFileSystem.java:1112) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.access$600(DistributedFileSystem.java:148) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem.java:1187) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem.java:1184) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listStatus(DistributedFileSystem.java:1194) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2078) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2122) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.CommonFSUtils.listStatus(CommonFSUtils.java:602) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.getFiles(AbstractFSWAL.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.close(AbstractFSWAL.java:1235) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.close0(AbstractFSWALProvider.java:170) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.cleanup(AbstractWALProvider.java:287) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.close(AbstractWALProvider.java:304) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.close(WALFactory.java:302) ~[classes/:?] at org.apache.hadoop.hbase.wal.TestWALFactory.tearDown(TestWALFactory.java:130) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:41:14,666 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testAppendClose Thread=269 (was 200) Potentially hanging thread: IPC Server handler 0 on default port 45919 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: qtp725983194-638 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@7fee8e7e java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp725983194-637 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server idle connection scanner for port 46873 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server idle connection scanner for port 38153 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@40f73ac java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-25-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 0 on default port 38153 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@44596e93 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:4689) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp1216210989-748 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data10/current/BP-1829792084-172.17.0.2-1733499651885 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-120-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 1 on default port 46873 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: pool-138-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 3 on default port 44581 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: IPC Server idle connection scanner for port 45919 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@1e3ef9a7 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:563) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp725983194-636-acceptor-0@2da8154-ServerConnector@495dc18a{HTTP/1.1, (http/1.1)}{localhost:41577} java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 2 on default port 44581 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@12c22dac[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 0 on default port 46873 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data7) java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656) Potentially hanging thread: qtp958717442-589 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb970445da8.run(Unknown Source) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46873 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-25-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-25-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Session-HouseKeeper-69f606a0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@7afa2375 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@7757d6c6 java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) java.base@17.0.11/sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:126) app//org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) app//org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:242) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server listener on 37105 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data9) java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656) Potentially hanging thread: Socket Reader #1 for port 37105 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497) app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476) Potentially hanging thread: qtp958717442-592 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp854269358-701 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp1216210989-746-acceptor-0@7675d052-ServerConnector@2129ddf3{HTTP/1.1, (http/1.1)}{localhost:41959} java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp958717442-593 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@5e4d1e0c java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp854269358-697 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb970445da8.run(Unknown Source) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (2120142737) connection to localhost/127.0.0.1:46873 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: qtp958717442-594 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 3 on default port 46873 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: qtp776016201-665 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb970445da8.run(Unknown Source) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 4 on default port 38153 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: IPC Server handler 2 on default port 34567 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: Session-HouseKeeper-4c592c4-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 3 on default port 45919 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: IPC Server handler 4 on default port 44581 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@38fb77d0[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 4 on default port 46873 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: pool-142-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp958717442-590-acceptor-0@454934be-ServerConnector@412c2abe{HTTP/1.1, (http/1.1)}{localhost:40627} java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp776016201-666-acceptor-0@2e4be799-ServerConnector@1148d54{HTTP/1.1, (http/1.1)}{localhost:38291} java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-125-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@2525b1f0 java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) java.base@17.0.11/sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:126) app//org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) app//org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:242) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34567 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp1216210989-745 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb970445da8.run(Unknown Source) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp462634889-793 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb970445da8.run(Unknown Source) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@bc32145 java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) java.base@17.0.11/sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:126) app//org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) app//org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:242) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@5ecf8f8e java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) java.base@17.0.11/sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:126) app//org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) app//org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:242) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp462634889-795 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp1216210989-747 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (2120142737) connection to localhost/127.0.0.1:34567 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: qtp725983194-635 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb970445da8.run(Unknown Source) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 0 on default port 34567 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: qtp776016201-668 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp958717442-588 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb970445da8.run(Unknown Source) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server idle connection scanner for port 34567 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@3cdaaaf0 java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) java.base@17.0.11/sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:126) app//org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) app//org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:242) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-104-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-109-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp854269358-699-acceptor-0@74071778-ServerConnector@14033e15{HTTP/1.1, (http/1.1)}{localhost:38819} java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data8/current/BP-1829792084-172.17.0.2-1733499651885 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 0 on default port 44581 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: Session-HouseKeeper-3a169f59-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-129-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@78fd008f java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@66935d9 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:4592) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@3a014b07 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:537) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-115-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 4 on default port 45919 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: pool-158-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp462634889-796 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-123-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 1 on default port 38153 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@58749f64 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp854269358-700 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp958717442-591 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb970445da8.run(Unknown Source) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 1 on default port 34567 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: nioEventLoopGroup-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 3 on default port 38153 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: IPC Server handler 4 on default port 34567 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: IPC Server handler 2 on default port 38153 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@197d0e4e java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:4550) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 1 on default port 45919 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: IPC Server handler 3 on default port 34567 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: qtp462634889-794-acceptor-0@430c1d86-ServerConnector@3237b321{HTTP/1.1, (http/1.1)}{localhost:34435} java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-132-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data7/current/BP-1829792084-172.17.0.2-1733499651885 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@7fe9b68e[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: CacheReplicationMonitor(1235004423) java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759) app//org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:186) Potentially hanging thread: Session-HouseKeeper-430e0f02-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data9/current/BP-1829792084-172.17.0.2-1733499651885 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor@5de88ad7 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor.run(PendingReconstructionBlocks.java:267) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-151-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Session-HouseKeeper-5e19b61c-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 2 on default port 45919 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: pool-145-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server idle connection scanner for port 44581 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: qtp776016201-667 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Session-HouseKeeper-4b20bb35-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@432e492[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-522905607_22 at /127.0.0.1:57118 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 2 on default port 46873 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: pool-164-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Server handler 1 on default port 44581 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) Potentially hanging thread: pool-155-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@e418108[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: qtp958717442-587 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb970445da8.run(Unknown Source) app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data8) java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656) Potentially hanging thread: pool-117-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-103-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data10) java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656) Potentially hanging thread: pool-112-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1219 (was 1155) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=526 (was 433) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9506 (was 10229) 2024-12-06T15:41:14,668 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1219 is superior to 1024 2024-12-06T15:41:14,691 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testEditAdd Thread=269, OpenFileDescriptor=1219, MaxFileDescriptor=1048576, SystemLoadAverage=526, ProcessCount=11, AvailableMemoryMB=9503 2024-12-06T15:41:14,691 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1219 is superior to 1024 2024-12-06T15:41:14,693 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:41:14,701 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T15:41:14,710 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-12-06T15:41:14,711 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=testeditadd%2C16010%2C1, suffix=, logDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testeditadd,16010,1, archiveDir=hdfs://localhost:37105/user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs, maxLogs=1760 2024-12-06T15:41:14,734 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testeditadd,16010,1/testeditadd%2C16010%2C1.1733499674712, exclude list is [], retry=0 2024-12-06T15:41:14,747 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37449,DS-877d5313-8b78-4ec1-91c4-7731b03c6bc8,DISK] 2024-12-06T15:41:14,753 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41809,DS-6e4baa1d-1227-4f3a-b14f-fac45062d8b6,DISK] 2024-12-06T15:41:14,765 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35821,DS-4f4b3052-a29f-49e8-a992-229f8202a569,DISK] 2024-12-06T15:41:14,799 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/WALs/testeditadd,16010,1/testeditadd%2C16010%2C1.1733499674712 2024-12-06T15:41:14,807 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43479:43479),(127.0.0.1/127.0.0.1:33349:33349),(127.0.0.1/127.0.0.1:34965:34965)] 2024-12-06T15:41:14,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41809 is added to blk_1073741855_1032 (size=514) 2024-12-06T15:41:14,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35821 is added to blk_1073741855_1032 (size=514) 2024-12-06T15:41:14,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37449 is added to blk_1073741855_1032 (size=514) 2024-12-06T15:41:14,835 INFO [Time-limited test {}] wal.TestWALFactory(515): testEditAdd/0324bb94cfed13c3672d5550eadc7c4d/2 [#edits: 10 = ] 2024-12-06T15:41:14,843 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fef6c9ed-9de7-5bae-1e96-a03bcd050886/oldWALs 2024-12-06T15:41:14,844 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL testeditadd%2C16010%2C1:(num 1733499674712) 2024-12-06T15:41:14,863 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testEditAdd Thread=272 (was 269) - Thread LEAK? -, OpenFileDescriptor=1255 (was 1219) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=526 (was 526), ProcessCount=11 (was 11), AvailableMemoryMB=9493 (was 9503) 2024-12-06T15:41:14,864 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1255 is superior to 1024 2024-12-06T15:41:14,879 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: wal.TestWALFactory#testWALProviders Thread=272, OpenFileDescriptor=1255, MaxFileDescriptor=1048576, SystemLoadAverage=526, ProcessCount=11, AvailableMemoryMB=9492 2024-12-06T15:41:14,879 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1255 is superior to 1024 2024-12-06T15:41:14,880 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:41:14,899 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:41:14,900 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:41:14,901 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:41:14,902 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:41:14,921 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: wal.TestWALFactory#testWALProviders Thread=272 (was 272), OpenFileDescriptor=1415 (was 1255) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=526 (was 526), ProcessCount=11 (was 11), AvailableMemoryMB=9488 (was 9492) 2024-12-06T15:41:14,921 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1415 is superior to 1024 2024-12-06T15:41:14,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-06T15:41:14,950 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@45a7078e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:41:14,950 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3237b321{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T15:41:14,950 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T15:41:14,951 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1fb841e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T15:41:14,951 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@53b6a0b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,STOPPED} 2024-12-06T15:41:14,954 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T15:41:14,954 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T15:41:14,954 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1829792084-172.17.0.2-1733499651885 (Datanode Uuid 08933cec-be98-46d5-86fb-b05c38673056) service to localhost/127.0.0.1:37105 2024-12-06T15:41:14,954 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T15:41:14,955 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data9/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:41:14,955 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T15:41:14,960 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data10/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:41:14,968 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@45b0aaa2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:41:14,969 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2129ddf3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T15:41:14,969 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T15:41:14,970 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64c6a8c9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T15:41:14,970 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@281828e9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,STOPPED} 2024-12-06T15:41:14,977 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T15:41:14,977 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T15:41:14,977 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1829792084-172.17.0.2-1733499651885 (Datanode Uuid de72905f-7ca5-4e87-92ad-8eb565185846) service to localhost/127.0.0.1:37105 2024-12-06T15:41:14,977 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T15:41:14,978 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data7/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:41:14,978 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data8/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:41:14,978 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T15:41:14,989 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2e5024e1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:41:14,990 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@14033e15{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T15:41:14,990 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T15:41:14,990 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2467ba64{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T15:41:14,990 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@383504c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,STOPPED} 2024-12-06T15:41:14,997 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T15:41:14,997 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T15:41:14,997 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1829792084-172.17.0.2-1733499651885 (Datanode Uuid 0d6e69cc-7877-4434-b2cb-608913e6b424) service to localhost/127.0.0.1:37105 2024-12-06T15:41:14,997 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T15:41:14,998 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data5/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:41:14,998 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data6/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:41:15,000 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T15:41:15,029 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@19bae41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:41:15,029 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1148d54{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T15:41:15,029 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T15:41:15,030 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c99e40{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T15:41:15,030 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c7fe737{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,STOPPED} 2024-12-06T15:41:15,033 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T15:41:15,033 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T15:41:15,033 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1829792084-172.17.0.2-1733499651885 (Datanode Uuid 466c1d1c-bc74-4274-ab94-ebed5ce6b316) service to localhost/127.0.0.1:37105 2024-12-06T15:41:15,033 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T15:41:15,034 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data3/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:41:15,034 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data4/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:41:15,035 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T15:41:15,053 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79863ca9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:41:15,053 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@495dc18a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T15:41:15,053 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T15:41:15,054 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ad6957a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T15:41:15,054 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6ab9b30d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,STOPPED} 2024-12-06T15:41:15,063 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T15:41:15,063 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T15:41:15,063 WARN [BP-1829792084-172.17.0.2-1733499651885 heartbeating to localhost/127.0.0.1:37105 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1829792084-172.17.0.2-1733499651885 (Datanode Uuid c262f2ad-b09a-4750-9e9a-43ee971188b0) service to localhost/127.0.0.1:37105 2024-12-06T15:41:15,063 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T15:41:15,064 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data1/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:41:15,064 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/cluster_94d6f1b2-05f0-e9a3-a4d1-78f3440ac225/data/data2/current/BP-1829792084-172.17.0.2-1733499651885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:41:15,065 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T15:41:15,075 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@754e9679{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T15:41:15,076 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@412c2abe{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T15:41:15,076 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T15:41:15,076 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79f425ce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T15:41:15,076 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39fa668d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/69a2fcac-31a8-f8ab-55ba-a106d9c0179d/hadoop.log.dir/,STOPPED} 2024-12-06T15:41:15,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down