2024-11-12 03:23:14,414 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-12 03:23:14,426 main DEBUG Took 0.009234 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 03:23:14,426 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 03:23:14,426 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 03:23:14,427 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 03:23:14,428 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 03:23:14,438 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 03:23:14,452 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,453 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 03:23:14,454 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,454 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 03:23:14,455 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,455 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 03:23:14,456 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,456 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 03:23:14,456 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,457 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 03:23:14,458 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,458 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 03:23:14,458 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,459 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 03:23:14,459 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,459 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 03:23:14,460 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,460 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 03:23:14,461 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,461 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 03:23:14,461 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,462 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 03:23:14,462 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,462 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 03:23:14,463 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,463 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 03:23:14,465 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 03:23:14,466 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 03:23:14,467 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 03:23:14,468 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 03:23:14,469 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 03:23:14,469 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 03:23:14,476 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 03:23:14,478 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 03:23:14,480 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 03:23:14,480 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 03:23:14,481 main DEBUG createAppenders(={Console}) 2024-11-12 03:23:14,482 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-11-12 03:23:14,482 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-12 03:23:14,482 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-11-12 03:23:14,483 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 03:23:14,483 main DEBUG OutputStream closed 2024-11-12 03:23:14,483 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 03:23:14,483 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 03:23:14,484 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2dc54ad4 OK 2024-11-12 03:23:14,557 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 03:23:14,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 03:23:14,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 03:23:14,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 03:23:14,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 03:23:14,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 03:23:14,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 03:23:14,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 03:23:14,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 03:23:14,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 03:23:14,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 03:23:14,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 03:23:14,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 03:23:14,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 03:23:14,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 03:23:14,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 03:23:14,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 03:23:14,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 03:23:14,568 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 03:23:14,569 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4f9a2c08) with optional ClassLoader: null 2024-11-12 03:23:14,569 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 03:23:14,570 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4f9a2c08] started OK. 2024-11-12T03:23:14,782 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9 2024-11-12 03:23:14,784 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 03:23:14,784 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T03:23:14,817 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStreamFSHLog timeout: 26 mins 2024-11-12T03:23:14,839 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1, deleteOnExit=true 2024-11-12T03:23:14,840 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/test.cache.data in system properties and HBase conf 2024-11-12T03:23:14,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T03:23:14,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/hadoop.log.dir in system properties and HBase conf 2024-11-12T03:23:14,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T03:23:14,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T03:23:14,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T03:23:14,950 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T03:23:15,060 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T03:23:15,064 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T03:23:15,064 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T03:23:15,065 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T03:23:15,065 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T03:23:15,066 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T03:23:15,066 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T03:23:15,067 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T03:23:15,067 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T03:23:15,068 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T03:23:15,068 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/nfs.dump.dir in system properties and HBase conf 2024-11-12T03:23:15,068 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/java.io.tmpdir in system properties and HBase conf 2024-11-12T03:23:15,069 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T03:23:15,069 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T03:23:15,070 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T03:23:16,352 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T03:23:16,418 INFO [Time-limited test {}] log.Log(170): Logging initialized @2741ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T03:23:16,492 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T03:23:16,567 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T03:23:16,587 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T03:23:16,587 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T03:23:16,589 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T03:23:16,601 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T03:23:16,603 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47b3659{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/hadoop.log.dir/,AVAILABLE} 2024-11-12T03:23:16,604 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cf0e64{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T03:23:16,848 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1460f442{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/java.io.tmpdir/jetty-localhost-32897-hadoop-hdfs-3_4_1-tests_jar-_-any-9619705082867008882/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T03:23:16,859 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@603bbd{HTTP/1.1, (http/1.1)}{localhost:32897} 2024-11-12T03:23:16,859 INFO [Time-limited test {}] server.Server(415): Started @3183ms 2024-11-12T03:23:17,567 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T03:23:17,580 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T03:23:17,581 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T03:23:17,581 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T03:23:17,582 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T03:23:17,583 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@229f9b5f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/hadoop.log.dir/,AVAILABLE} 2024-11-12T03:23:17,585 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54ac3606{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T03:23:17,717 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bc6d0f6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/java.io.tmpdir/jetty-localhost-38315-hadoop-hdfs-3_4_1-tests_jar-_-any-17451268129472899333/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T03:23:17,718 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c1a1027{HTTP/1.1, (http/1.1)}{localhost:38315} 2024-11-12T03:23:17,718 INFO [Time-limited test {}] server.Server(415): Started @4042ms 2024-11-12T03:23:17,797 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T03:23:18,209 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T03:23:18,228 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T03:23:18,270 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T03:23:18,271 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T03:23:18,271 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T03:23:18,272 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b6067f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/hadoop.log.dir/,AVAILABLE} 2024-11-12T03:23:18,274 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@212998b8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T03:23:18,420 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@71aaceb5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/java.io.tmpdir/jetty-localhost-38967-hadoop-hdfs-3_4_1-tests_jar-_-any-14858555739854872172/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T03:23:18,421 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@39c5da84{HTTP/1.1, (http/1.1)}{localhost:38967} 2024-11-12T03:23:18,421 INFO [Time-limited test {}] server.Server(415): Started @4745ms 2024-11-12T03:23:18,425 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T03:23:18,496 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T03:23:18,505 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T03:23:18,509 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T03:23:18,509 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T03:23:18,509 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T03:23:18,510 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c21a271{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/hadoop.log.dir/,AVAILABLE} 2024-11-12T03:23:18,511 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32e0ec32{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T03:23:18,622 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@55b5b1fa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/java.io.tmpdir/jetty-localhost-38071-hadoop-hdfs-3_4_1-tests_jar-_-any-17012330817000984247/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T03:23:18,623 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2bdee3f9{HTTP/1.1, (http/1.1)}{localhost:38071} 2024-11-12T03:23:18,623 INFO [Time-limited test {}] server.Server(415): Started @4946ms 2024-11-12T03:23:18,625 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T03:23:19,829 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data4/current/BP-1788124332-172.17.0.2-1731381795654/current, will proceed with Du for space computation calculation, 2024-11-12T03:23:19,829 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data3/current/BP-1788124332-172.17.0.2-1731381795654/current, will proceed with Du for space computation calculation, 2024-11-12T03:23:19,829 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data1/current/BP-1788124332-172.17.0.2-1731381795654/current, will proceed with Du for space computation calculation, 2024-11-12T03:23:19,829 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data2/current/BP-1788124332-172.17.0.2-1731381795654/current, will proceed with Du for space computation calculation, 2024-11-12T03:23:19,864 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T03:23:19,864 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T03:23:19,913 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x57ce998b9ce993f8 with lease ID 0x7930595e3f8d7b0f: Processing first storage report for DS-c40206bf-7c73-473f-a859-addea61a74b8 from datanode DatanodeRegistration(127.0.0.1:33481, datanodeUuid=f9295c0b-a302-42a3-ab9e-7aef280a8e6e, infoPort=36985, infoSecurePort=0, ipcPort=41311, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654) 2024-11-12T03:23:19,914 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x57ce998b9ce993f8 with lease ID 0x7930595e3f8d7b0f: from storage DS-c40206bf-7c73-473f-a859-addea61a74b8 node DatanodeRegistration(127.0.0.1:33481, datanodeUuid=f9295c0b-a302-42a3-ab9e-7aef280a8e6e, infoPort=36985, infoSecurePort=0, ipcPort=41311, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T03:23:19,914 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf5ef5eba5fb9586b with lease ID 0x7930595e3f8d7b10: Processing first storage report for DS-487adbaa-b861-4c3e-97a0-db0f39f142dd from datanode DatanodeRegistration(127.0.0.1:40231, datanodeUuid=6596d77a-3a69-4811-8164-e228ce265e02, infoPort=42817, infoSecurePort=0, ipcPort=43729, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654) 2024-11-12T03:23:19,914 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf5ef5eba5fb9586b with lease ID 0x7930595e3f8d7b10: from storage DS-487adbaa-b861-4c3e-97a0-db0f39f142dd node DatanodeRegistration(127.0.0.1:40231, datanodeUuid=6596d77a-3a69-4811-8164-e228ce265e02, infoPort=42817, infoSecurePort=0, ipcPort=43729, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T03:23:19,915 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x57ce998b9ce993f8 with lease ID 0x7930595e3f8d7b0f: Processing first storage report for DS-21d51b25-00fa-4316-b381-fb717ca59909 from datanode DatanodeRegistration(127.0.0.1:33481, datanodeUuid=f9295c0b-a302-42a3-ab9e-7aef280a8e6e, infoPort=36985, infoSecurePort=0, ipcPort=41311, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654) 2024-11-12T03:23:19,915 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x57ce998b9ce993f8 with lease ID 0x7930595e3f8d7b0f: from storage DS-21d51b25-00fa-4316-b381-fb717ca59909 node DatanodeRegistration(127.0.0.1:33481, datanodeUuid=f9295c0b-a302-42a3-ab9e-7aef280a8e6e, infoPort=36985, infoSecurePort=0, ipcPort=41311, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T03:23:19,915 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf5ef5eba5fb9586b with lease ID 0x7930595e3f8d7b10: Processing first storage report for DS-1841f4f4-cfff-454d-b217-879dcaa63068 from datanode DatanodeRegistration(127.0.0.1:40231, datanodeUuid=6596d77a-3a69-4811-8164-e228ce265e02, infoPort=42817, infoSecurePort=0, ipcPort=43729, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654) 2024-11-12T03:23:19,915 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf5ef5eba5fb9586b with lease ID 0x7930595e3f8d7b10: from storage DS-1841f4f4-cfff-454d-b217-879dcaa63068 node DatanodeRegistration(127.0.0.1:40231, datanodeUuid=6596d77a-3a69-4811-8164-e228ce265e02, infoPort=42817, infoSecurePort=0, ipcPort=43729, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T03:23:19,932 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data6/current/BP-1788124332-172.17.0.2-1731381795654/current, will proceed with Du for space computation calculation, 2024-11-12T03:23:19,932 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data5/current/BP-1788124332-172.17.0.2-1731381795654/current, will proceed with Du for space computation calculation, 2024-11-12T03:23:19,953 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T03:23:19,958 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf78bd402d11a0ca0 with lease ID 0x7930595e3f8d7b11: Processing first storage report for DS-467b940c-b06f-48ed-b767-610c9104ada2 from datanode DatanodeRegistration(127.0.0.1:46807, datanodeUuid=7c1c332d-7003-4dc6-8192-de0ecd333db6, infoPort=39813, infoSecurePort=0, ipcPort=46613, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654) 2024-11-12T03:23:19,958 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf78bd402d11a0ca0 with lease ID 0x7930595e3f8d7b11: from storage DS-467b940c-b06f-48ed-b767-610c9104ada2 node DatanodeRegistration(127.0.0.1:46807, datanodeUuid=7c1c332d-7003-4dc6-8192-de0ecd333db6, infoPort=39813, infoSecurePort=0, ipcPort=46613, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T03:23:19,959 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf78bd402d11a0ca0 with lease ID 0x7930595e3f8d7b11: Processing first storage report for DS-b0f1ceb5-e396-4dea-8b67-c55862e40fdf from datanode DatanodeRegistration(127.0.0.1:46807, datanodeUuid=7c1c332d-7003-4dc6-8192-de0ecd333db6, infoPort=39813, infoSecurePort=0, ipcPort=46613, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654) 2024-11-12T03:23:19,959 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf78bd402d11a0ca0 with lease ID 0x7930595e3f8d7b11: from storage DS-b0f1ceb5-e396-4dea-8b67-c55862e40fdf node DatanodeRegistration(127.0.0.1:46807, datanodeUuid=7c1c332d-7003-4dc6-8192-de0ecd333db6, infoPort=39813, infoSecurePort=0, ipcPort=46613, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T03:23:20,057 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9 2024-11-12T03:23:20,070 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[0: isCompressionEnabled=false] Thread=157, OpenFileDescriptor=391, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=7064 2024-11-12T03:23:20,073 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T03:23:20,584 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T03:23:20,599 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:20,677 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:20,720 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381800714 2024-11-12T03:23:20,900 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381800714 2024-11-12T03:23:20,961 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:21,018 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381800714, startPosition=0, beingWritten=true 2024-11-12T03:23:21,161 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381800714 to pos 203, reset compression=false 2024-11-12T03:23:21,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:21,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:21,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:21,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:21,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:21,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741825_1001 (size=447) 2024-11-12T03:23:21,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741825_1001 (size=447) 2024-11-12T03:23:21,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741825_1001 (size=447) 2024-11-12T03:23:21,614 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:21,617 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381800714) 2024-11-12T03:23:21,625 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[0: isCompressionEnabled=false] Thread=167 (was 157) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: LeaseRenewer:jenkins@localhost:38535 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@4b0094bc java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=407 (was 391) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 469), ProcessCount=11 (was 11), AvailableMemoryMB=6962 (was 7064) 2024-11-12T03:23:21,632 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[0: isCompressionEnabled=false] Thread=167, OpenFileDescriptor=407, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=6961 2024-11-12T03:23:21,633 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:21,639 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:21,642 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381801641 2024-11-12T03:23:21,654 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381801641 2024-11-12T03:23:21,655 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813)] 2024-11-12T03:23:21,658 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381801641, startPosition=0, beingWritten=true 2024-11-12T03:23:21,737 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-12T03:23:21,741 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381801641, startPosition=0, beingWritten=true 2024-11-12T03:23:21,754 WARN [Thread-175 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-12T03:23:21,778 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381801641, startPosition=0, beingWritten=true 2024-11-12T03:23:21,793 WARN [Thread-175 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-12T03:23:21,824 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381801641, startPosition=0, beingWritten=true 2024-11-12T03:23:21,839 WARN [Thread-175 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-12T03:23:21,884 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381801641, startPosition=0, beingWritten=true 2024-11-12T03:23:21,896 DEBUG [Thread-175 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-12T03:23:21,897 DEBUG [Thread-175 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381801641 to pos 460, reset compression=false 2024-11-12T03:23:21,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:21,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:21,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:21,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:21,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:21,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741826_1002 (size=468) 2024-11-12T03:23:21,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741826_1002 (size=468) 2024-11-12T03:23:21,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741826_1002 (size=468) 2024-11-12T03:23:21,921 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:21,922 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381801641) 2024-11-12T03:23:21,924 DEBUG [Thread-175 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381801641 to pos 460, reset compression=false 2024-11-12T03:23:21,929 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[0: isCompressionEnabled=false] Thread=175 (was 167) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-175 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=415 (was 407) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 469), ProcessCount=11 (was 11), AvailableMemoryMB=6942 (was 6961) 2024-11-12T03:23:21,934 INFO [Thread-175 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381801641 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381801641 2024-11-12T03:23:21,937 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[0: isCompressionEnabled=false] Thread=175, OpenFileDescriptor=415, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=6942 2024-11-12T03:23:21,939 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:21,940 DEBUG [Thread-175 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381801641 2024-11-12T03:23:21,941 DEBUG [Thread-175 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:21,941 DEBUG [Thread-175 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:21,945 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:21,949 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D.1731381801948 2024-11-12T03:23:21,961 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_ isCompressionEnabled=false]/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D.1731381801948 2024-11-12T03:23:21,964 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:22,117 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor log.1.false 2024-11-12T03:23:22,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741828_1004 (size=231) 2024-11-12T03:23:22,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741828_1004 (size=231) 2024-11-12T03:23:22,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741828_1004 (size=231) 2024-11-12T03:23:22,147 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-12T03:23:22,148 DEBUG [Thread-199 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/user/jenkins/log.1.false, startPosition=0, beingWritten=false 2024-11-12T03:23:22,157 DEBUG [Thread-199 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/user/jenkins/log.1.false 2024-11-12T03:23:22,157 DEBUG [Thread-199 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-12T03:23:22,158 DEBUG [Thread-199 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/user/jenkins/log.2.false, startPosition=0, beingWritten=false 2024-11-12T03:23:22,161 WARN [Thread-199 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:38535/user/jenkins/log.2.false, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-12T03:23:22,161 DEBUG [Thread-199 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/user/jenkins/log.2.false 2024-11-12T03:23:22,162 DEBUG [Thread-199 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:22,162 DEBUG [Thread-199 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:22,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741827_1003 (size=93) 2024-11-12T03:23:22,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741827_1003 (size=93) 2024-11-12T03:23:22,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741827_1003 (size=93) 2024-11-12T03:23:22,173 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:22,173 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381801948) 2024-11-12T03:23:22,182 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[0: isCompressionEnabled=false] Thread=180 (was 175) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 415) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 469), ProcessCount=11 (was 11), AvailableMemoryMB=6907 (was 6942) 2024-11-12T03:23:22,190 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[0: isCompressionEnabled=false] Thread=180, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=6907 2024-11-12T03:23:22,191 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:22,203 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:22,206 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802206 2024-11-12T03:23:22,235 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802206 2024-11-12T03:23:22,265 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:22,268 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802267 2024-11-12T03:23:22,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,322 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802267 2024-11-12T03:23:22,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741829_1005 (size=93) 2024-11-12T03:23:22,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741829_1005 (size=93) 2024-11-12T03:23:22,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741829_1005 (size=93) 2024-11-12T03:23:22,340 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:22,340 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802206 to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802206 2024-11-12T03:23:22,342 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:22,344 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802206 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802206 2024-11-12T03:23:22,361 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-12T03:23:22,369 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:22,370 DEBUG [Thread-216 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802206, startPosition=0, beingWritten=false 2024-11-12T03:23:22,377 INFO [Thread-216 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802206 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802206 2024-11-12T03:23:22,381 WARN [Thread-216 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802206, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-12T03:23:22,382 DEBUG [Thread-216 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802206 2024-11-12T03:23:22,382 DEBUG [Thread-216 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802267, startPosition=0, beingWritten=true 2024-11-12T03:23:22,403 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802267 to pos 0, reset compression=false 2024-11-12T03:23:22,427 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802267 to pos 0, reset compression=false 2024-11-12T03:23:22,449 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802267 to pos 0, reset compression=false 2024-11-12T03:23:22,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,471 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802267 to pos 0, reset compression=false 2024-11-12T03:23:22,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741830_1006 (size=93) 2024-11-12T03:23:22,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741830_1006 (size=93) 2024-11-12T03:23:22,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741830_1006 (size=93) 2024-11-12T03:23:22,490 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:22,490 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381802267) 2024-11-12T03:23:22,494 INFO [Thread-216 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802267 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802267 2024-11-12T03:23:22,501 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[0: isCompressionEnabled=false] Thread=186 (was 180) Potentially hanging thread: Thread-216 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy46.getFileInfo(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getFileInfo$41(ClientNamenodeProtocolTranslatorPB.java:820) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$314/0x00007f01587db988.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:820) jdk.internal.reflect.GeneratedMethodAccessor30.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy47.getFileInfo(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1770) app//org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1828) app//org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1825) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1840) app//org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1862) app//org.apache.hadoop.hbase.wal.AbstractFSWALProvider.findArchivedLog(AbstractFSWALProvider.java:534) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.openArchivedWAL(AbstractProtobufWALReader.java:183) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:196) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:212) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 469), ProcessCount=11 (was 11), AvailableMemoryMB=6878 (was 6907) 2024-11-12T03:23:22,507 DEBUG [Thread-216 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731381802267 2024-11-12T03:23:22,508 DEBUG [Thread-216 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:22,508 DEBUG [Thread-216 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:22,512 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[0: isCompressionEnabled=false] Thread=186, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=6877 2024-11-12T03:23:22,514 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:22,522 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:22,526 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731381802525 2024-11-12T03:23:22,578 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731381802525 2024-11-12T03:23:22,605 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:22,626 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731381802626 2024-11-12T03:23:22,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,669 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731381802525 with entries=10, filesize=1.30 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731381802626 2024-11-12T03:23:22,674 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:22,675 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731381802525 is not closed yet, will try archiving it next time 2024-11-12T03:23:22,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741831_1007 (size=1343) 2024-11-12T03:23:22,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741831_1007 (size=1343) 2024-11-12T03:23:22,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741831_1007 (size=1343) 2024-11-12T03:23:22,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741832_1008 (size=718) 2024-11-12T03:23:22,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741832_1008 (size=718) 2024-11-12T03:23:22,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741832_1008 (size=718) 2024-11-12T03:23:22,692 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=10, replicationBatchQueueCapacity=1 2024-11-12T03:23:22,693 DEBUG [Thread-238 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731381802525, startPosition=0, beingWritten=false 2024-11-12T03:23:22,715 DEBUG [Thread-238 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-12T03:23:22,716 DEBUG [Thread-238 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731381802525 2024-11-12T03:23:22,717 DEBUG [Thread-238 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731381802626, startPosition=0, beingWritten=false 2024-11-12T03:23:22,729 DEBUG [Thread-238 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731381802626 2024-11-12T03:23:22,729 DEBUG [Thread-238 {}] regionserver.ReplicationSourceWALReader(177): Read 5 WAL entries eligible for replication 2024-11-12T03:23:22,730 DEBUG [Thread-238 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:22,730 DEBUG [Thread-238 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:22,740 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:22,740 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381802626) 2024-11-12T03:23:22,752 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[0: isCompressionEnabled=false] Thread=191 (was 186) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 469), ProcessCount=11 (was 11), AvailableMemoryMB=6853 (was 6877) 2024-11-12T03:23:22,761 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[0: isCompressionEnabled=false] Thread=191, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=6852 2024-11-12T03:23:22,763 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:22,771 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionForRecoveredQueue_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:22,773 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D.1731381802773 2024-11-12T03:23:22,798 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionForRecoveredQueue_0_ isCompressionEnabled=false]/testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D.1731381802773 2024-11-12T03:23:22,804 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:22,829 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-12T03:23:22,834 DEBUG [Thread-248 {}] regionserver.WALEntryStream(254): Creating new reader emptyLog, startPosition=0, beingWritten=false 2024-11-12T03:23:22,836 WARN [Thread-248 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: emptyLog, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-12T03:23:22,837 DEBUG [Thread-248 {}] regionserver.WALEntryStream(456): EOF, closing emptyLog 2024-11-12T03:23:22,837 DEBUG [Thread-248 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:22,837 DEBUG [Thread-248 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:22,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741833_1009 (size=93) 2024-11-12T03:23:22,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741833_1009 (size=93) 2024-11-12T03:23:22,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741833_1009 (size=93) 2024-11-12T03:23:22,857 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:22,857 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381802773) 2024-11-12T03:23:22,868 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[0: isCompressionEnabled=false] Thread=196 (was 191) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 469), ProcessCount=11 (was 11), AvailableMemoryMB=6843 (was 6852) 2024-11-12T03:23:22,879 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[0: isCompressionEnabled=false] Thread=196, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=6842 2024-11-12T03:23:22,883 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:22,889 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:22,891 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802890 2024-11-12T03:23:22,906 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802890 2024-11-12T03:23:22,913 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:22,923 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802922 2024-11-12T03:23:22,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,946 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802890 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802922 2024-11-12T03:23:22,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741834_1010 (size=218) 2024-11-12T03:23:22,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741834_1010 (size=218) 2024-11-12T03:23:22,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741834_1010 (size=218) 2024-11-12T03:23:22,951 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:22,956 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:22,959 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-12T03:23:22,959 DEBUG [Thread-265 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802890, startPosition=0, beingWritten=false 2024-11-12T03:23:22,964 DEBUG [Thread-265 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802890 2024-11-12T03:23:22,965 DEBUG [Thread-265 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-12T03:23:22,965 DEBUG [Thread-265 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802922, startPosition=0, beingWritten=true 2024-11-12T03:23:22,980 DEBUG [Thread-265 {}] regionserver.ReplicationSourceWALReader(177): Read 20 WAL entries eligible for replication 2024-11-12T03:23:22,981 DEBUG [Thread-265 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802922 to pos 2585, reset compression=false 2024-11-12T03:23:22,981 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802981 2024-11-12T03:23:22,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:22,991 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802922 with entries=20, filesize=2.52 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802981 2024-11-12T03:23:22,992 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:22,992 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802922 is not closed yet, will try archiving it next time 2024-11-12T03:23:22,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741835_1011 (size=2593) 2024-11-12T03:23:22,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741835_1011 (size=2593) 2024-11-12T03:23:22,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741835_1011 (size=2593) 2024-11-12T03:23:23,013 DEBUG [Thread-265 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802922 to pos 2585, reset compression=false 2024-11-12T03:23:23,017 DEBUG [Thread-265 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802922 2024-11-12T03:23:23,018 DEBUG [Thread-265 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802981, startPosition=0, beingWritten=true 2024-11-12T03:23:23,032 DEBUG [Thread-265 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-12T03:23:23,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,033 DEBUG [Thread-265 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802981 to pos 1335, reset compression=false 2024-11-12T03:23:23,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741836_1012 (size=1343) 2024-11-12T03:23:23,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741836_1012 (size=1343) 2024-11-12T03:23:23,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741836_1012 (size=1343) 2024-11-12T03:23:23,044 DEBUG [Thread-265 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381802981 2024-11-12T03:23:23,044 DEBUG [Thread-265 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:23,044 DEBUG [Thread-265 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:23,045 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:23,045 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381802981) 2024-11-12T03:23:23,051 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[0: isCompressionEnabled=false] Thread=202 (was 196) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-265 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:343) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.replicationDone(ReplicationSourceWALReader.java:123) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:151) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 469), ProcessCount=11 (was 11), AvailableMemoryMB=6833 (was 6842) 2024-11-12T03:23:23,059 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[0: isCompressionEnabled=false] Thread=202, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=6833 2024-11-12T03:23:23,060 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:23,064 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALKeySerialization_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:23,065 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D.1731381803065 2024-11-12T03:23:23,076 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALKeySerialization_0_ isCompressionEnabled=false]/testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D.1731381803065 2024-11-12T03:23:23,079 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:23,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741837_1013 (size=93) 2024-11-12T03:23:23,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741837_1013 (size=93) 2024-11-12T03:23:23,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741837_1013 (size=93) 2024-11-12T03:23:23,099 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:23,099 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381803065) 2024-11-12T03:23:23,112 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[0: isCompressionEnabled=false] Thread=207 (was 202) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 469), ProcessCount=11 (was 11), AvailableMemoryMB=6831 (was 6833) 2024-11-12T03:23:23,121 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[0: isCompressionEnabled=false] Thread=207, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=6830 2024-11-12T03:23:23,122 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:23,127 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:23,128 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731381803128 2024-11-12T03:23:23,158 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731381803128 2024-11-12T03:23:23,164 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:23,172 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731381803128, startPosition=0, beingWritten=true 2024-11-12T03:23:23,187 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-12T03:23:23,191 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:23,291 DEBUG [Thread-295 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731381803128, startPosition=0, beingWritten=true 2024-11-12T03:23:23,300 DEBUG [Thread-295 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-12T03:23:23,300 DEBUG [Thread-295 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731381803128 to pos 460, reset compression=false 2024-11-12T03:23:23,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741838_1014 (size=468) 2024-11-12T03:23:23,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741838_1014 (size=468) 2024-11-12T03:23:23,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741838_1014 (size=468) 2024-11-12T03:23:23,309 DEBUG [Thread-295 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731381803128 2024-11-12T03:23:23,309 DEBUG [Thread-295 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:23,309 DEBUG [Thread-295 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:23,310 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:23,310 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381803128) 2024-11-12T03:23:23,318 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[0: isCompressionEnabled=false] Thread=215 (was 207) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-295 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:343) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.replicationDone(ReplicationSourceWALReader.java:123) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:151) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 469), ProcessCount=11 (was 11), AvailableMemoryMB=6817 (was 6830) 2024-11-12T03:23:23,326 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[0: isCompressionEnabled=false] Thread=215, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=6817 2024-11-12T03:23:23,327 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:23,332 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:23,333 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381803333 2024-11-12T03:23:23,342 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381803333 2024-11-12T03:23:23,343 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:23,347 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:23,347 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381803333, startPosition=0, beingWritten=true 2024-11-12T03:23:23,355 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381803354 2024-11-12T03:23:23,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,365 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381803333 with entries=3, filesize=439 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381803354 2024-11-12T03:23:23,367 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813)] 2024-11-12T03:23:23,367 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381803333 is not closed yet, will try archiving it next time 2024-11-12T03:23:23,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741839_1015 (size=447) 2024-11-12T03:23:23,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741839_1015 (size=447) 2024-11-12T03:23:23,369 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:23,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741839_1015 (size=447) 2024-11-12T03:23:23,369 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:23,369 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381803333 for the last time to pos 321, reset compression=false 2024-11-12T03:23:23,373 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:23,373 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381803333 2024-11-12T03:23:23,474 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731381803354, startPosition=0, beingWritten=true 2024-11-12T03:23:23,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741840_1016 (size=211) 2024-11-12T03:23:23,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741840_1016 (size=211) 2024-11-12T03:23:23,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741840_1016 (size=211) 2024-11-12T03:23:23,495 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:23,496 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381803354) 2024-11-12T03:23:23,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[0: isCompressionEnabled=false] Thread=220 (was 215) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 469), ProcessCount=11 (was 11), AvailableMemoryMB=6810 (was 6817) 2024-11-12T03:23:23,510 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[0: isCompressionEnabled=false] Thread=220, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=6810 2024-11-12T03:23:23,511 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:23,515 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:23,515 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731381803515 2024-11-12T03:23:23,523 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731381803515 2024-11-12T03:23:23,524 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:23,530 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:23,530 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731381803515, startPosition=0, beingWritten=true 2024-11-12T03:23:23,537 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731381803537 2024-11-12T03:23:23,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,546 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731381803515 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731381803537 2024-11-12T03:23:23,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741841_1017 (size=218) 2024-11-12T03:23:23,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741841_1017 (size=218) 2024-11-12T03:23:23,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741841_1017 (size=218) 2024-11-12T03:23:23,560 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:23,562 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:23,563 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731381803515 for the last time to pos 210, reset compression=false 2024-11-12T03:23:23,568 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731381803515 and length of the file is 218 2024-11-12T03:23:23,568 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731381803515 2024-11-12T03:23:23,668 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731381803537, startPosition=0, beingWritten=true 2024-11-12T03:23:23,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:23,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741842_1018 (size=218) 2024-11-12T03:23:23,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741842_1018 (size=218) 2024-11-12T03:23:23,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741842_1018 (size=218) 2024-11-12T03:23:23,691 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:23,691 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381803537) 2024-11-12T03:23:23,699 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[0: isCompressionEnabled=false] Thread=225 (was 220) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 469), ProcessCount=11 (was 11), AvailableMemoryMB=6797 (was 6810) 2024-11-12T03:23:23,707 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[0: isCompressionEnabled=false] Thread=225, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=6797 2024-11-12T03:23:23,709 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:23,713 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:23,714 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731381803714 2024-11-12T03:23:23,723 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731381803714 2024-11-12T03:23:23,724 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:23,732 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731381803714, startPosition=0, beingWritten=true 2024-11-12T03:23:23,742 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 272 and read up to 320 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:612) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.KeyValueUtil.createKeyValueFromInputStream(KeyValueUtil.java:659) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.KeyValueCodecWithTags$KeyValueDecoder.parseCell(KeyValueCodecWithTags.java:77) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-12T03:23:24,743 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731381803714 to pos 203, reset compression=false 2024-11-12T03:23:24,750 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 272 and read up to 320 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:615) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.KeyValueUtil.createKeyValueFromInputStream(KeyValueUtil.java:659) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.KeyValueCodecWithTags$KeyValueDecoder.parseCell(KeyValueCodecWithTags.java:77) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-12T03:23:24,751 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731381803714 to pos 203, reset compression=false 2024-11-12T03:23:24,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741843_1019 (size=329) 2024-11-12T03:23:24,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741843_1019 (size=329) 2024-11-12T03:23:24,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741843_1019 (size=329) 2024-11-12T03:23:24,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:24,765 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381803714) 2024-11-12T03:23:24,774 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[0: isCompressionEnabled=false] Thread=230 (was 225) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 469) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6756 (was 6797) 2024-11-12T03:23:24,781 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[0: isCompressionEnabled=false] Thread=230, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6755 2024-11-12T03:23:24,782 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:24,788 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:24,789 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381804789 2024-11-12T03:23:24,797 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381804789 2024-11-12T03:23:24,800 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813)] 2024-11-12T03:23:24,806 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381804789, startPosition=0, beingWritten=true 2024-11-12T03:23:24,815 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381804789, startPosition=203, beingWritten=true 2024-11-12T03:23:24,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741844_1020 (size=447) 2024-11-12T03:23:24,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741844_1020 (size=447) 2024-11-12T03:23:24,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741844_1020 (size=447) 2024-11-12T03:23:24,831 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:24,831 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381804789) 2024-11-12T03:23:24,839 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[0: isCompressionEnabled=false] Thread=235 (was 230) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6751 (was 6755) 2024-11-12T03:23:24,847 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[0: isCompressionEnabled=false] Thread=235, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6751 2024-11-12T03:23:24,848 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:24,852 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:24,852 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731381804852 2024-11-12T03:23:24,861 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731381804852 2024-11-12T03:23:24,865 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:24,868 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731381804852, startPosition=0, beingWritten=true 2024-11-12T03:23:24,876 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-12T03:23:24,876 DEBUG [Thread-364 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731381804852, startPosition=0, beingWritten=true 2024-11-12T03:23:24,884 DEBUG [Thread-364 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-12T03:23:24,884 DEBUG [Thread-364 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731381804852 to pos 460, reset compression=false 2024-11-12T03:23:24,891 DEBUG [Thread-364 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-12T03:23:24,891 DEBUG [Thread-364 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731381804852 to pos 580, reset compression=false 2024-11-12T03:23:24,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741845_1021 (size=588) 2024-11-12T03:23:24,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741845_1021 (size=588) 2024-11-12T03:23:24,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741845_1021 (size=588) 2024-11-12T03:23:24,897 DEBUG [Thread-364 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731381804852 2024-11-12T03:23:24,898 DEBUG [Thread-364 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:24,898 DEBUG [Thread-364 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:24,899 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:24,899 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381804852) 2024-11-12T03:23:24,909 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[0: isCompressionEnabled=false] Thread=241 (was 235) Potentially hanging thread: Thread-364 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:343) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.replicationDone(ReplicationSourceWALReader.java:123) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:151) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6745 (was 6751) 2024-11-12T03:23:24,918 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[0: isCompressionEnabled=false] Thread=241, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6744 2024-11-12T03:23:24,919 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:24,922 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:24,923 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 2024-11-12T03:23:24,932 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 2024-11-12T03:23:24,936 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:24,940 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804939 2024-11-12T03:23:24,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:24,951 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804939 2024-11-12T03:23:24,953 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:24,953 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 is not closed yet, will try archiving it next time 2024-11-12T03:23:24,954 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:24,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741846_1022 (size=93) 2024-11-12T03:23:24,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741846_1022 (size=93) 2024-11-12T03:23:24,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741846_1022 (size=93) 2024-11-12T03:23:24,958 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 2024-11-12T03:23:25,057 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 2024-11-12T03:23:25,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741848_1024 (size=85) 2024-11-12T03:23:25,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741848_1024 (size=85) 2024-11-12T03:23:25,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741848_1024 (size=85) 2024-11-12T03:23:25,083 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923, startPosition=0, beingWritten=false 2024-11-12T03:23:25,085 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 2024-11-12T03:23:25,090 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 for the last time to pos 0, reset compression=false 2024-11-12T03:23:25,092 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 2024-11-12T03:23:25,097 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 2024-11-12T03:23:25,098 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 and length of the file is 85 2024-11-12T03:23:25,098 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804923 2024-11-12T03:23:25,098 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731381804939, startPosition=0, beingWritten=true 2024-11-12T03:23:25,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741847_1023 (size=218) 2024-11-12T03:23:25,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741847_1023 (size=218) 2024-11-12T03:23:25,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741847_1023 (size=218) 2024-11-12T03:23:25,112 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:25,112 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381804939) 2024-11-12T03:23:25,119 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[0: isCompressionEnabled=false] Thread=246 (was 241) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6733 (was 6744) 2024-11-12T03:23:25,126 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[0: isCompressionEnabled=false] Thread=246, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6732 2024-11-12T03:23:25,128 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:25,131 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:25,132 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132 2024-11-12T03:23:25,141 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132 2024-11-12T03:23:25,145 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:25,148 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132, startPosition=0, beingWritten=true 2024-11-12T03:23:25,156 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-12T03:23:25,157 DEBUG [Thread-402 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132, startPosition=0, beingWritten=true 2024-11-12T03:23:25,163 WARN [Thread-402 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-12T03:23:25,184 DEBUG [Thread-402 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132, startPosition=0, beingWritten=true 2024-11-12T03:23:25,193 WARN [Thread-402 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-12T03:23:25,225 DEBUG [Thread-402 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132, startPosition=0, beingWritten=true 2024-11-12T03:23:25,236 WARN [Thread-402 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-12T03:23:25,277 DEBUG [Thread-402 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132, startPosition=0, beingWritten=true 2024-11-12T03:23:25,284 WARN [Thread-402 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-12T03:23:25,335 DEBUG [Thread-402 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132, startPosition=0, beingWritten=true 2024-11-12T03:23:25,345 WARN [Thread-402 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-12T03:23:25,406 DEBUG [Thread-402 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132, startPosition=0, beingWritten=true 2024-11-12T03:23:25,418 DEBUG [Thread-402 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-12T03:23:25,419 DEBUG [Thread-402 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132 to pos 460, reset compression=false 2024-11-12T03:23:25,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741849_1025 (size=468) 2024-11-12T03:23:25,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741849_1025 (size=468) 2024-11-12T03:23:25,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741849_1025 (size=468) 2024-11-12T03:23:25,430 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:25,430 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381805132) 2024-11-12T03:23:25,438 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[0: isCompressionEnabled=false] Thread=252 (was 246) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-402 java.base@17.0.11/java.util.zip.Inflater.inflateBytesBytes(Native Method) java.base@17.0.11/java.util.zip.Inflater.inflate(Inflater.java:378) java.base@17.0.11/java.util.zip.InflaterInputStream.read(InflaterInputStream.java:152) java.base@17.0.11/jdk.internal.loader.Resource.getBytes(Resource.java:126) java.base@17.0.11/jdk.internal.loader.URLClassPath$JarLoader$2.getBytes(URLClassPath.java:893) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:859) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:773) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:768) java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:318) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:781) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639) java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) java.base@17.0.11/java.lang.ClassLoader.loadClass(ClassLoader.java:525) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.(WALProtos.java:15199) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.internalGetFieldAccessorTable(WALProtos.java:1640) app//org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.getDescriptorForType(GeneratedMessage.java:106) app//org.apache.hbase.thirdparty.com.google.protobuf.MessageReflection.findMissingFields(MessageReflection.java:151) app//org.apache.hbase.thirdparty.com.google.protobuf.MessageReflection.findMissingFields(MessageReflection.java:185) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage$Builder.newUninitializedMessageException(AbstractMessage.java:446) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage.newUninitializedMessageException(AbstractMessage.java:284) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.newUninitializedMessageException(AbstractParser.java:30) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:44) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) app//org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6719 (was 6732) 2024-11-12T03:23:25,445 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[0: isCompressionEnabled=false] Thread=252, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6717 2024-11-12T03:23:25,447 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:25,455 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEmptyStream_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:25,456 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1731381805456 2024-11-12T03:23:25,489 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEmptyStream_0_ isCompressionEnabled=false]/testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1731381805456 2024-11-12T03:23:25,498 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:25,509 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEmptyStream_0_ isCompressionEnabled=false]/testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1731381805456, startPosition=0, beingWritten=true 2024-11-12T03:23:25,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741850_1026 (size=93) 2024-11-12T03:23:25,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741850_1026 (size=93) 2024-11-12T03:23:25,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741850_1026 (size=93) 2024-11-12T03:23:25,530 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:25,530 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381805456) 2024-11-12T03:23:25,538 INFO [Thread-402 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=460, currentPosition=468 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-12T03:23:25,538 DEBUG [Thread-402 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132 for the last time to pos 460, reset compression=false 2024-11-12T03:23:25,540 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[0: isCompressionEnabled=false] Thread=258 (was 252) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=429 (was 427) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6699 (was 6717) 2024-11-12T03:23:25,540 INFO [Thread-402 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132 2024-11-12T03:23:25,550 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[0: isCompressionEnabled=false] Thread=258, OpenFileDescriptor=429, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6697 2024-11-12T03:23:25,551 INFO [Thread-402 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132 2024-11-12T03:23:25,551 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:25,551 DEBUG [Thread-402 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132 and length of the file is 468 2024-11-12T03:23:25,551 DEBUG [Thread-402 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731381805132 2024-11-12T03:23:25,552 DEBUG [Thread-402 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:25,552 DEBUG [Thread-402 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:25,556 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:25,557 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731381805557 2024-11-12T03:23:25,570 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731381805557 2024-11-12T03:23:25,571 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:25,574 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731381805557, startPosition=0, beingWritten=true 2024-11-12T03:23:25,583 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:25,584 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731381805557, startPosition=210, beingWritten=true 2024-11-12T03:23:25,593 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731381805592 2024-11-12T03:23:25,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,603 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731381805557 with entries=3, filesize=460 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731381805592 2024-11-12T03:23:25,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741851_1027 (size=468) 2024-11-12T03:23:25,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741851_1027 (size=468) 2024-11-12T03:23:25,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741851_1027 (size=468) 2024-11-12T03:23:25,616 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:25,619 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:25,619 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731381805557, startPosition=335, beingWritten=false 2024-11-12T03:23:25,623 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:25,623 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731381805557 2024-11-12T03:23:25,723 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731381805592, startPosition=0, beingWritten=true 2024-11-12T03:23:25,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741852_1028 (size=218) 2024-11-12T03:23:25,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741852_1028 (size=218) 2024-11-12T03:23:25,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741852_1028 (size=218) 2024-11-12T03:23:25,737 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:25,737 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381805592) 2024-11-12T03:23:25,748 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[0: isCompressionEnabled=false] Thread=262 (was 258) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #35] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 429) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6676 (was 6697) 2024-11-12T03:23:25,757 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[0: isCompressionEnabled=false] Thread=262, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6675 2024-11-12T03:23:25,759 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:25,763 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:25,765 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381805764 2024-11-12T03:23:25,778 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381805764 2024-11-12T03:23:25,780 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813)] 2024-11-12T03:23:25,787 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381805764, startPosition=0, beingWritten=true 2024-11-12T03:23:25,794 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731381805764, startPosition=210, beingWritten=true 2024-11-12T03:23:25,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741853_1029 (size=468) 2024-11-12T03:23:25,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741853_1029 (size=468) 2024-11-12T03:23:25,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741853_1029 (size=468) 2024-11-12T03:23:25,807 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:25,807 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381805764) 2024-11-12T03:23:25,818 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[0: isCompressionEnabled=false] Thread=267 (was 262) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6665 (was 6675) 2024-11-12T03:23:25,827 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[0: isCompressionEnabled=false] Thread=267, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6663 2024-11-12T03:23:25,828 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:25,832 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:25,833 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731381805833 2024-11-12T03:23:25,840 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731381805833 2024-11-12T03:23:25,844 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:25,847 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731381805847 2024-11-12T03:23:25,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,855 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731381805833 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731381805847 2024-11-12T03:23:25,857 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:25,857 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731381805833 is not closed yet, will try archiving it next time 2024-11-12T03:23:25,858 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:25,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741854_1030 (size=218) 2024-11-12T03:23:25,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741854_1030 (size=218) 2024-11-12T03:23:25,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741854_1030 (size=218) 2024-11-12T03:23:25,867 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731381805833, startPosition=0, beingWritten=false 2024-11-12T03:23:25,871 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731381805833 2024-11-12T03:23:25,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741855_1031 (size=93) 2024-11-12T03:23:25,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741855_1031 (size=93) 2024-11-12T03:23:25,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741855_1031 (size=93) 2024-11-12T03:23:25,879 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:25,879 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D:(num 1731381805847) 2024-11-12T03:23:25,890 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[0: isCompressionEnabled=false] Thread=272 (was 267) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6655 (was 6663) 2024-11-12T03:23:25,901 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[1: isCompressionEnabled=true] Thread=272, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6653 2024-11-12T03:23:25,902 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:25,905 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:25,906 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381805906 2024-11-12T03:23:25,917 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381805906 2024-11-12T03:23:25,921 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:25,923 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381805906, startPosition=0, beingWritten=true 2024-11-12T03:23:25,928 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381805906: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:25,932 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381805906 to pos 197, reset compression=false 2024-11-12T03:23:25,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:25,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741856_1032 (size=317) 2024-11-12T03:23:25,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741856_1032 (size=317) 2024-11-12T03:23:25,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741856_1032 (size=317) 2024-11-12T03:23:25,946 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:25,946 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381805906) 2024-11-12T03:23:25,956 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[1: isCompressionEnabled=true] Thread=278 (was 272) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6648 (was 6653) 2024-11-12T03:23:25,966 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[1: isCompressionEnabled=true] Thread=278, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6647 2024-11-12T03:23:25,967 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:25,972 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:25,973 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972 2024-11-12T03:23:25,982 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972 2024-11-12T03:23:25,983 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813)] 2024-11-12T03:23:25,985 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972, startPosition=0, beingWritten=true 2024-11-12T03:23:25,990 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:25,993 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-12T03:23:25,995 DEBUG [Thread-485 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972, startPosition=0, beingWritten=true 2024-11-12T03:23:26,006 DEBUG [Thread-485 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:26,009 WARN [Thread-485 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-12T03:23:26,029 DEBUG [Thread-485 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972, startPosition=0, beingWritten=true 2024-11-12T03:23:26,034 DEBUG [Thread-485 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:26,036 WARN [Thread-485 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-12T03:23:26,067 DEBUG [Thread-485 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972, startPosition=0, beingWritten=true 2024-11-12T03:23:26,075 DEBUG [Thread-485 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:26,079 WARN [Thread-485 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-12T03:23:26,096 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T03:23:26,120 DEBUG [Thread-485 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972, startPosition=0, beingWritten=true 2024-11-12T03:23:26,125 DEBUG [Thread-485 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:26,127 DEBUG [Thread-485 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-12T03:23:26,127 DEBUG [Thread-485 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972 to pos 322, reset compression=false 2024-11-12T03:23:26,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741857_1033 (size=330) 2024-11-12T03:23:26,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741857_1033 (size=330) 2024-11-12T03:23:26,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741857_1033 (size=330) 2024-11-12T03:23:26,144 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:26,144 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381805972) 2024-11-12T03:23:26,152 DEBUG [Thread-485 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972 to pos 322, reset compression=false 2024-11-12T03:23:26,152 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[1: isCompressionEnabled=true] Thread=285 (was 278) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-485 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6632 (was 6647) 2024-11-12T03:23:26,157 INFO [Thread-485 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972 2024-11-12T03:23:26,160 DEBUG [Thread-485 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381805972 2024-11-12T03:23:26,160 DEBUG [Thread-485 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:26,160 DEBUG [Thread-485 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:26,162 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[1: isCompressionEnabled=true] Thread=285, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6631 2024-11-12T03:23:26,163 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:26,166 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:26,167 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D.1731381806167 2024-11-12T03:23:26,175 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_ isCompressionEnabled=true]/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D.1731381806167 2024-11-12T03:23:26,175 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813)] 2024-11-12T03:23:26,182 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor log.1.true 2024-11-12T03:23:26,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741859_1035 (size=225) 2024-11-12T03:23:26,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741859_1035 (size=225) 2024-11-12T03:23:26,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741859_1035 (size=225) 2024-11-12T03:23:26,196 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-12T03:23:26,197 DEBUG [Thread-509 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/user/jenkins/log.1.true, startPosition=0, beingWritten=false 2024-11-12T03:23:26,199 DEBUG [Thread-509 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/user/jenkins/log.1.true: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:26,201 DEBUG [Thread-509 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/user/jenkins/log.1.true 2024-11-12T03:23:26,201 DEBUG [Thread-509 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-12T03:23:26,201 DEBUG [Thread-509 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/user/jenkins/log.2.true, startPosition=0, beingWritten=false 2024-11-12T03:23:26,203 WARN [Thread-509 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:38535/user/jenkins/log.2.true, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-12T03:23:26,203 DEBUG [Thread-509 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/user/jenkins/log.2.true 2024-11-12T03:23:26,203 DEBUG [Thread-509 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:26,203 DEBUG [Thread-509 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:26,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741858_1034 (size=93) 2024-11-12T03:23:26,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741858_1034 (size=93) 2024-11-12T03:23:26,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741858_1034 (size=93) 2024-11-12T03:23:26,209 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:26,210 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381806167) 2024-11-12T03:23:26,218 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[1: isCompressionEnabled=true] Thread=290 (was 285) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6623 (was 6631) 2024-11-12T03:23:26,227 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[1: isCompressionEnabled=true] Thread=290, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6623 2024-11-12T03:23:26,228 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:26,231 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:26,231 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806231 2024-11-12T03:23:26,238 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806231 2024-11-12T03:23:26,239 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:26,240 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806239 2024-11-12T03:23:26,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,251 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806239 2024-11-12T03:23:26,252 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813)] 2024-11-12T03:23:26,252 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806231 is not closed yet, will try archiving it next time 2024-11-12T03:23:26,252 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:26,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741860_1036 (size=93) 2024-11-12T03:23:26,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741860_1036 (size=93) 2024-11-12T03:23:26,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741860_1036 (size=93) 2024-11-12T03:23:26,256 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806231 to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806231 2024-11-12T03:23:26,353 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806231 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806231 2024-11-12T03:23:26,357 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-12T03:23:26,358 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:26,358 DEBUG [Thread-526 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806231, startPosition=0, beingWritten=false 2024-11-12T03:23:26,360 INFO [Thread-526 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806231 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806231 2024-11-12T03:23:26,361 WARN [Thread-526 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806231, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-12T03:23:26,361 DEBUG [Thread-526 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806231 2024-11-12T03:23:26,362 DEBUG [Thread-526 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806239, startPosition=0, beingWritten=true 2024-11-12T03:23:26,366 DEBUG [Thread-526 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806239: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:26,377 DEBUG [Thread-526 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806239 to pos 0, reset compression=false 2024-11-12T03:23:26,394 DEBUG [Thread-526 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806239 to pos 0, reset compression=false 2024-11-12T03:23:26,410 DEBUG [Thread-526 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806239 to pos 0, reset compression=false 2024-11-12T03:23:26,426 DEBUG [Thread-526 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806239 to pos 0, reset compression=false 2024-11-12T03:23:26,442 DEBUG [Thread-526 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806239 to pos 0, reset compression=false 2024-11-12T03:23:26,458 DEBUG [Thread-526 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806239 to pos 0, reset compression=false 2024-11-12T03:23:26,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741861_1037 (size=93) 2024-11-12T03:23:26,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741861_1037 (size=93) 2024-11-12T03:23:26,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741861_1037 (size=93) 2024-11-12T03:23:26,466 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:26,466 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381806239) 2024-11-12T03:23:26,469 DEBUG [Thread-526 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731381806239 2024-11-12T03:23:26,469 DEBUG [Thread-526 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:26,470 DEBUG [Thread-526 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:26,474 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[1: isCompressionEnabled=true] Thread=296 (was 290) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #16] java.base@17.0.11/java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:237) java.base@17.0.11/java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) app//org.apache.hadoop.net.SocketIOWithTimeout.waitForIO(SocketIOWithTimeout.java:244) app//org.apache.hadoop.net.SocketOutputStream.waitForWritable(SocketOutputStream.java:171) app//org.apache.hadoop.net.SocketOutputStream.transferToFully(SocketOutputStream.java:219) app//org.apache.hadoop.hdfs.server.datanode.FileIoProvider.transferToSocketFully(FileIoProvider.java:278) app//org.apache.hadoop.hdfs.server.datanode.BlockSender.sendPacket(BlockSender.java:619) app//org.apache.hadoop.hdfs.server.datanode.BlockSender.doSendBlock(BlockSender.java:819) app//org.apache.hadoop.hdfs.server.datanode.BlockSender.sendBlock(BlockSender.java:766) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:621) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-526 app//org.apache.logging.slf4j.Log4jLogger.isTraceEnabled(Log4jLogger.java:84) app//org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.read(BlockReaderRemote.java:136) app//org.apache.hadoop.hdfs.ByteArrayStrategy.readFromBlock(ReaderStrategy.java:118) app//org.apache.hadoop.hdfs.DFSInputStream.readBuffer(DFSInputStream.java:789) app//org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:855) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) java.base@17.0.11/java.io.DataInputStream.readInt(DataInputStream.java:381) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:212) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6609 (was 6623) 2024-11-12T03:23:26,483 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[1: isCompressionEnabled=true] Thread=296, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6608 2024-11-12T03:23:26,484 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:26,487 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:26,488 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731381806487 2024-11-12T03:23:26,498 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731381806487 2024-11-12T03:23:26,499 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813)] 2024-11-12T03:23:26,503 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731381806503 2024-11-12T03:23:26,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,513 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731381806487 with entries=10, filesize=715 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731381806503 2024-11-12T03:23:26,517 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:26,517 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731381806487 is not closed yet, will try archiving it next time 2024-11-12T03:23:26,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741862_1038 (size=723) 2024-11-12T03:23:26,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741862_1038 (size=723) 2024-11-12T03:23:26,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741862_1038 (size=723) 2024-11-12T03:23:26,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741863_1039 (size=440) 2024-11-12T03:23:26,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741863_1039 (size=440) 2024-11-12T03:23:26,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741863_1039 (size=440) 2024-11-12T03:23:26,525 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=10, replicationBatchQueueCapacity=1 2024-11-12T03:23:26,526 DEBUG [Thread-550 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731381806487, startPosition=0, beingWritten=false 2024-11-12T03:23:26,528 DEBUG [Thread-550 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731381806487: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:26,531 DEBUG [Thread-550 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-12T03:23:26,531 DEBUG [Thread-550 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731381806487 2024-11-12T03:23:26,532 DEBUG [Thread-550 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731381806503, startPosition=0, beingWritten=false 2024-11-12T03:23:26,534 DEBUG [Thread-550 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731381806503: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:26,537 DEBUG [Thread-550 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731381806503 2024-11-12T03:23:26,537 DEBUG [Thread-550 {}] regionserver.ReplicationSourceWALReader(177): Read 5 WAL entries eligible for replication 2024-11-12T03:23:26,537 DEBUG [Thread-550 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:26,537 DEBUG [Thread-550 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:26,540 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:26,540 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381806503) 2024-11-12T03:23:26,549 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[1: isCompressionEnabled=true] Thread=301 (was 296) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #49] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6597 (was 6608) 2024-11-12T03:23:26,557 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[1: isCompressionEnabled=true] Thread=301, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6596 2024-11-12T03:23:26,558 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:26,561 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionForRecoveredQueue_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:26,561 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D.1731381806561 2024-11-12T03:23:26,568 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEOFExceptionForRecoveredQueue_1_ isCompressionEnabled=true]/testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D.1731381806561 2024-11-12T03:23:26,569 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:26,575 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-12T03:23:26,576 DEBUG [Thread-560 {}] regionserver.WALEntryStream(254): Creating new reader emptyLog, startPosition=0, beingWritten=false 2024-11-12T03:23:26,577 WARN [Thread-560 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: emptyLog, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-12T03:23:26,577 DEBUG [Thread-560 {}] regionserver.WALEntryStream(456): EOF, closing emptyLog 2024-11-12T03:23:26,578 DEBUG [Thread-560 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:26,578 DEBUG [Thread-560 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:26,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741864_1040 (size=93) 2024-11-12T03:23:26,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741864_1040 (size=93) 2024-11-12T03:23:26,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741864_1040 (size=93) 2024-11-12T03:23:26,584 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:26,584 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381806561) 2024-11-12T03:23:26,593 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[1: isCompressionEnabled=true] Thread=306 (was 301) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6591 (was 6596) 2024-11-12T03:23:26,604 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[1: isCompressionEnabled=true] Thread=306, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6590 2024-11-12T03:23:26,605 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:26,608 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:26,609 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806609 2024-11-12T03:23:26,617 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806609 2024-11-12T03:23:26,620 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813)] 2024-11-12T03:23:26,626 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806626 2024-11-12T03:23:26,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,642 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806609 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806626 2024-11-12T03:23:26,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741865_1041 (size=212) 2024-11-12T03:23:26,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741865_1041 (size=212) 2024-11-12T03:23:26,646 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:26,646 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806609 is not closed yet, will try archiving it next time 2024-11-12T03:23:26,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741865_1041 (size=212) 2024-11-12T03:23:26,650 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:26,753 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-12T03:23:26,754 DEBUG [Thread-577 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806609, startPosition=0, beingWritten=false 2024-11-12T03:23:26,757 DEBUG [Thread-577 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806609: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:26,758 DEBUG [Thread-577 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806609 2024-11-12T03:23:26,758 DEBUG [Thread-577 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-12T03:23:26,758 DEBUG [Thread-577 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806626, startPosition=0, beingWritten=true 2024-11-12T03:23:26,763 DEBUG [Thread-577 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806626: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:26,768 DEBUG [Thread-577 {}] regionserver.ReplicationSourceWALReader(177): Read 20 WAL entries eligible for replication 2024-11-12T03:23:26,768 DEBUG [Thread-577 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806626 to pos 1273, reset compression=false 2024-11-12T03:23:26,769 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806769 2024-11-12T03:23:26,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,777 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806626 with entries=20, filesize=1.24 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806769 2024-11-12T03:23:26,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741866_1042 (size=1281) 2024-11-12T03:23:26,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741866_1042 (size=1281) 2024-11-12T03:23:26,780 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:26,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741866_1042 (size=1281) 2024-11-12T03:23:26,780 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806626 is not closed yet, will try archiving it next time 2024-11-12T03:23:26,801 DEBUG [Thread-577 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806626 to pos 1273, reset compression=false 2024-11-12T03:23:26,804 DEBUG [Thread-577 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806626 2024-11-12T03:23:26,805 DEBUG [Thread-577 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806769, startPosition=0, beingWritten=true 2024-11-12T03:23:26,809 DEBUG [Thread-577 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806769: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:26,813 DEBUG [Thread-577 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-12T03:23:26,813 DEBUG [Thread-577 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806769 to pos 715, reset compression=false 2024-11-12T03:23:26,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741867_1043 (size=723) 2024-11-12T03:23:26,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741867_1043 (size=723) 2024-11-12T03:23:26,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741867_1043 (size=723) 2024-11-12T03:23:26,819 DEBUG [Thread-577 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381806769 2024-11-12T03:23:26,819 DEBUG [Thread-577 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:26,819 DEBUG [Thread-577 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:26,821 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:26,821 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381806769) 2024-11-12T03:23:26,829 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[1: isCompressionEnabled=true] Thread=312 (was 306) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #53] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-577 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:343) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.replicationDone(ReplicationSourceWALReader.java:123) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:151) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #46] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Waiting for operation #46] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6569 (was 6590) 2024-11-12T03:23:26,838 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[1: isCompressionEnabled=true] Thread=312, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6568 2024-11-12T03:23:26,838 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:26,841 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALKeySerialization_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:26,842 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D.1731381806841 2024-11-12T03:23:26,849 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALKeySerialization_1_ isCompressionEnabled=true]/testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D.1731381806841 2024-11-12T03:23:26,850 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:26,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:26,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741868_1044 (size=93) 2024-11-12T03:23:26,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741868_1044 (size=93) 2024-11-12T03:23:26,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741868_1044 (size=93) 2024-11-12T03:23:26,856 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:26,856 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381806841) 2024-11-12T03:23:26,866 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[1: isCompressionEnabled=true] Thread=317 (was 312) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6564 (was 6568) 2024-11-12T03:23:26,875 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[1: isCompressionEnabled=true] Thread=317, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6563 2024-11-12T03:23:26,875 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:26,878 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:26,879 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879 2024-11-12T03:23:26,886 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879 2024-11-12T03:23:26,887 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:26,889 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879, startPosition=0, beingWritten=true 2024-11-12T03:23:26,894 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:26,896 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-12T03:23:26,899 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:27,001 DEBUG [Thread-607 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879, startPosition=0, beingWritten=true 2024-11-12T03:23:27,006 DEBUG [Thread-607 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:27,008 DEBUG [Thread-607 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-12T03:23:27,008 DEBUG [Thread-607 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879 to pos 314, reset compression=false 2024-11-12T03:23:27,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741869_1045 (size=322) 2024-11-12T03:23:27,013 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Sending block BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045] {}] datanode.DataXceiver(658): DatanodeRegistration(127.0.0.1:40231, datanodeUuid=6596d77a-3a69-4811-8164-e228ce265e02, infoPort=42817, infoSecurePort=0, ipcPort=43729, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654):Got exception while serving BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045 to /127.0.0.1:37938 java.io.FileNotFoundException: BlockId 1073741869 is not valid. at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:810) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:801) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getMetaDataInputStream(FsDatasetImpl.java:246) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:319) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T03:23:27,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741869_1045 (size=322) 2024-11-12T03:23:27,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741869_1045 (size=322) 2024-11-12T03:23:27,014 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:37938 [Sending block BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045] {}] datanode.DataXceiver(331): 127.0.0.1:40231:DataXceiver error processing READ_BLOCK operation src: /127.0.0.1:37938 dst: /127.0.0.1:40231 java.io.FileNotFoundException: BlockId 1073741869 is not valid. at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:810) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:801) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getMetaDataInputStream(FsDatasetImpl.java:246) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:319) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T03:23:27,015 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57166 [Sending block BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045] {}] datanode.DataXceiver(658): DatanodeRegistration(127.0.0.1:40231, datanodeUuid=6596d77a-3a69-4811-8164-e228ce265e02, infoPort=42817, infoSecurePort=0, ipcPort=43729, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654):Got exception while serving BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045 to /127.0.0.1:57166 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T03:23:27,015 WARN [Thread-607 {}] impl.BlockReaderFactory(772): I/O error constructing remote block reader for block BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045 java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:57166, remote=/127.0.0.1:40231, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879, for pool BP-1788124332-172.17.0.2-1731381795654 block 1073741869_1045 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-12T03:23:27,015 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57166 [Sending block BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045] {}] datanode.DataXceiver(331): 127.0.0.1:40231:DataXceiver error processing READ_BLOCK operation src: /127.0.0.1:57166 dst: /127.0.0.1:40231 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T03:23:27,015 WARN [Thread-607 {}] hdfs.DFSInputStream(667): Failed to connect to /127.0.0.1:40231 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879 for block BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045, add to deadNodes and continue. java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:57166, remote=/127.0.0.1:40231, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879, for pool BP-1788124332-172.17.0.2-1731381795654 block 1073741869_1045 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-12T03:23:27,016 DEBUG [Thread-607 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879 for the last time to pos 314, reset compression=false 2024-11-12T03:23:27,017 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:27,017 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381806879) 2024-11-12T03:23:27,021 INFO [Thread-607 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879 2024-11-12T03:23:27,023 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57176 [Sending block BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045] {}] datanode.DataXceiver(658): DatanodeRegistration(127.0.0.1:40231, datanodeUuid=6596d77a-3a69-4811-8164-e228ce265e02, infoPort=42817, infoSecurePort=0, ipcPort=43729, storageInfo=lv=-57;cid=testClusterID;nsid=1318625501;c=1731381795654):Got exception while serving BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045 to /127.0.0.1:57176 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T03:23:27,023 WARN [Thread-607 {}] impl.BlockReaderFactory(772): I/O error constructing remote block reader for block BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045 java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:57176, remote=/127.0.0.1:40231, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879, for pool BP-1788124332-172.17.0.2-1731381795654 block 1073741869_1045 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-12T03:23:27,023 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57176 [Sending block BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045] {}] datanode.DataXceiver(331): 127.0.0.1:40231:DataXceiver error processing READ_BLOCK operation src: /127.0.0.1:57176 dst: /127.0.0.1:40231 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T03:23:27,023 WARN [Thread-607 {}] hdfs.DFSInputStream(667): Failed to connect to /127.0.0.1:40231 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879 for block BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045, add to deadNodes and continue. java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1788124332-172.17.0.2-1731381795654:blk_1073741869_1045. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:57176, remote=/127.0.0.1:40231, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879, for pool BP-1788124332-172.17.0.2-1731381795654 block 1073741869_1045 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-12T03:23:27,025 INFO [Thread-607 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879 2024-11-12T03:23:27,026 DEBUG [Thread-607 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879 and length of the file is 322 2024-11-12T03:23:27,026 DEBUG [Thread-607 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731381806879 2024-11-12T03:23:27,026 DEBUG [Thread-607 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:27,026 DEBUG [Thread-607 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:27,026 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[1: isCompressionEnabled=true] Thread=323 (was 317) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #55] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Thread-607 java.base@17.0.11/javax.security.auth.Subject.getSubject(Subject.java:300) app//org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:588) app//org.apache.hadoop.fs.FileSystem$Cache$Key.(FileSystem.java:3888) app//org.apache.hadoop.fs.FileSystem$Cache$Key.(FileSystem.java:3878) app//org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3666) app//org.apache.hadoop.fs.FileSystem.get(FileSystem.java:557) app//org.apache.hadoop.fs.FileSystem.get(FileSystem.java:289) app//org.apache.hadoop.fs.FileSystem.get(FileSystem.java:541) app//org.apache.hadoop.fs.Path.getFileSystem(Path.java:366) app//org.apache.hadoop.hbase.util.CommonFSUtils.isValidWALRootDir(CommonFSUtils.java:359) app//org.apache.hadoop.hbase.util.CommonFSUtils.getWALRootDir(CommonFSUtils.java:322) app//org.apache.hadoop.hbase.wal.AbstractFSWALProvider.findArchivedLog(AbstractFSWALProvider.java:529) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.openArchivedWAL(AbstractProtobufWALReader.java:183) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:196) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) - Thread LEAK? -, OpenFileDescriptor=429 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6553 (was 6563) 2024-11-12T03:23:27,035 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[1: isCompressionEnabled=true] Thread=323, OpenFileDescriptor=429, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6553 2024-11-12T03:23:27,035 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:27,039 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:27,039 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381807039 2024-11-12T03:23:27,047 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381807039 2024-11-12T03:23:27,047 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:27,050 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:27,050 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381807039, startPosition=0, beingWritten=true 2024-11-12T03:23:27,055 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381807039: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:27,058 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381807057 2024-11-12T03:23:27,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,066 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381807039 with entries=3, filesize=309 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381807057 2024-11-12T03:23:27,067 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:27,068 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381807039 is not closed yet, will try archiving it next time 2024-11-12T03:23:27,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741870_1046 (size=317) 2024-11-12T03:23:27,070 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:27,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741870_1046 (size=317) 2024-11-12T03:23:27,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741870_1046 (size=317) 2024-11-12T03:23:27,071 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:27,071 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381807039 for the last time to pos 253, reset compression=false 2024-11-12T03:23:27,075 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:27,075 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381807039 2024-11-12T03:23:27,176 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381807057, startPosition=0, beingWritten=true 2024-11-12T03:23:27,186 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731381807057: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:27,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741871_1047 (size=205) 2024-11-12T03:23:27,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741871_1047 (size=205) 2024-11-12T03:23:27,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741871_1047 (size=205) 2024-11-12T03:23:27,197 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:27,197 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381807057) 2024-11-12T03:23:27,207 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[1: isCompressionEnabled=true] Thread=329 (was 323) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57202 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 429) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6544 (was 6553) 2024-11-12T03:23:27,215 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[1: isCompressionEnabled=true] Thread=329, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6542 2024-11-12T03:23:27,216 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:27,219 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:27,220 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731381807219 2024-11-12T03:23:27,227 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731381807219 2024-11-12T03:23:27,231 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:27,234 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:27,234 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731381807219, startPosition=0, beingWritten=true 2024-11-12T03:23:27,238 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731381807219: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:27,240 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731381807240 2024-11-12T03:23:27,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,249 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731381807219 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731381807240 2024-11-12T03:23:27,251 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:27,252 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731381807219 is not closed yet, will try archiving it next time 2024-11-12T03:23:27,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741872_1048 (size=212) 2024-11-12T03:23:27,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741872_1048 (size=212) 2024-11-12T03:23:27,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741872_1048 (size=212) 2024-11-12T03:23:27,255 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:27,255 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731381807219 for the last time to pos 204, reset compression=false 2024-11-12T03:23:27,258 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731381807219 and length of the file is 212 2024-11-12T03:23:27,258 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731381807219 2024-11-12T03:23:27,359 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731381807240, startPosition=0, beingWritten=true 2024-11-12T03:23:27,371 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731381807240: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:27,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:27,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741873_1049 (size=212) 2024-11-12T03:23:27,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741873_1049 (size=212) 2024-11-12T03:23:27,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741873_1049 (size=212) 2024-11-12T03:23:27,381 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:27,381 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381807240) 2024-11-12T03:23:27,391 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[1: isCompressionEnabled=true] Thread=334 (was 329) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #57] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57202 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6534 (was 6542) 2024-11-12T03:23:27,400 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[1: isCompressionEnabled=true] Thread=334, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6533 2024-11-12T03:23:27,400 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:27,403 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:27,404 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731381807404 2024-11-12T03:23:27,410 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731381807404 2024-11-12T03:23:27,411 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:27,413 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731381807404, startPosition=0, beingWritten=true 2024-11-12T03:23:27,418 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731381807404: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:27,420 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 225 and read up to 252 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:612) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALCellCodec$CompressedKvDecoder.parseCell(WALCellCodec.java:340) ~[classes/:?] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-12T03:23:28,421 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731381807404 to pos 197, reset compression=true 2024-11-12T03:23:28,429 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 225 and read up to 252 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:615) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALCellCodec$CompressedKvDecoder.parseCell(WALCellCodec.java:340) ~[classes/:?] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-12T03:23:28,429 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731381807404 to pos 197, reset compression=true 2024-11-12T03:23:28,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741874_1050 (size=261) 2024-11-12T03:23:28,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741874_1050 (size=261) 2024-11-12T03:23:28,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741874_1050 (size=261) 2024-11-12T03:23:28,451 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:28,451 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381807404) 2024-11-12T03:23:28,470 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[1: isCompressionEnabled=true] Thread=340 (was 334) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #59] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57202 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: FSHLog-0-hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9-prefix:testReadBeyondCommittedLength_1_ isCompressionEnabled=true] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #48] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6438 (was 6533) 2024-11-12T03:23:28,481 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[1: isCompressionEnabled=true] Thread=339, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6437 2024-11-12T03:23:28,482 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:28,485 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:28,485 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381808485 2024-11-12T03:23:28,493 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381808485 2024-11-12T03:23:28,497 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:28,499 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381808485, startPosition=0, beingWritten=true 2024-11-12T03:23:28,504 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381808485: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:28,508 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381808485, startPosition=197, beingWritten=true 2024-11-12T03:23:28,512 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381808485: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:28,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741875_1051 (size=317) 2024-11-12T03:23:28,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741875_1051 (size=317) 2024-11-12T03:23:28,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741875_1051 (size=317) 2024-11-12T03:23:28,520 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:28,520 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381808485) 2024-11-12T03:23:28,529 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[1: isCompressionEnabled=true] Thread=344 (was 339) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #61] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57202 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6434 (was 6437) 2024-11-12T03:23:28,539 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[1: isCompressionEnabled=true] Thread=344, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6433 2024-11-12T03:23:28,539 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:28,542 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:28,543 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543 2024-11-12T03:23:28,553 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543 2024-11-12T03:23:28,554 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:28,556 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543, startPosition=0, beingWritten=true 2024-11-12T03:23:28,561 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:28,564 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-12T03:23:28,564 DEBUG [Thread-679 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543, startPosition=0, beingWritten=true 2024-11-12T03:23:28,569 DEBUG [Thread-679 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:28,571 DEBUG [Thread-679 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-12T03:23:28,571 DEBUG [Thread-679 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543 to pos 314, reset compression=false 2024-11-12T03:23:28,576 DEBUG [Thread-679 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-12T03:23:28,576 DEBUG [Thread-679 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543 to pos 372, reset compression=false 2024-11-12T03:23:28,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741876_1052 (size=380) 2024-11-12T03:23:28,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741876_1052 (size=380) 2024-11-12T03:23:28,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741876_1052 (size=380) 2024-11-12T03:23:28,583 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:28,583 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381808543) 2024-11-12T03:23:28,583 INFO [Thread-679 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=372, currentPosition=380 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-12T03:23:28,586 DEBUG [Thread-679 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543 for the last time to pos 372, reset compression=false 2024-11-12T03:23:28,587 INFO [Thread-679 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543 2024-11-12T03:23:28,590 INFO [Thread-679 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543 2024-11-12T03:23:28,591 DEBUG [Thread-679 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543 and length of the file is 380 2024-11-12T03:23:28,591 DEBUG [Thread-679 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731381808543 2024-11-12T03:23:28,591 DEBUG [Thread-679 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:28,591 DEBUG [Thread-679 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:28,593 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[1: isCompressionEnabled=true] Thread=350 (was 344) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #63] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57202 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-679 java.base@17.0.11/jdk.internal.reflect.Reflection.getCallerClass(Native Method) java.base@17.0.11/java.lang.Class.getClassLoader(Class.java:901) app//org.apache.logging.log4j.core.impl.ThrowableProxyHelper.toCacheEntry(ThrowableProxyHelper.java:184) app//org.apache.logging.log4j.core.impl.ThrowableProxyHelper.toExtendedStackTrace(ThrowableProxyHelper.java:112) app//org.apache.logging.log4j.core.impl.ThrowableProxy.(ThrowableProxy.java:113) app//org.apache.logging.log4j.core.impl.ThrowableProxy.(ThrowableProxy.java:97) app//org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:629) app//org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:63) app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:384) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:375) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:216) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:59) app//org.apache.logging.log4j.core.layout.AbstractStringLayout.toByteArray(AbstractStringLayout.java:294) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:145) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2028) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1891) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:152) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #51] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6427 (was 6433) 2024-11-12T03:23:28,605 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[1: isCompressionEnabled=true] Thread=350, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=6426 2024-11-12T03:23:28,605 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:28,608 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:28,609 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 2024-11-12T03:23:28,617 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 2024-11-12T03:23:28,621 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:28,625 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808624 2024-11-12T03:23:28,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,634 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808624 2024-11-12T03:23:28,637 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:28,637 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 is not closed yet, will try archiving it next time 2024-11-12T03:23:28,638 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:28,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741877_1053 (size=93) 2024-11-12T03:23:28,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741877_1053 (size=93) 2024-11-12T03:23:28,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741877_1053 (size=93) 2024-11-12T03:23:28,640 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 2024-11-12T03:23:28,740 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 2024-11-12T03:23:28,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741879_1055 (size=85) 2024-11-12T03:23:28,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741879_1055 (size=85) 2024-11-12T03:23:28,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741879_1055 (size=85) 2024-11-12T03:23:28,750 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609, startPosition=0, beingWritten=false 2024-11-12T03:23:28,752 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 2024-11-12T03:23:28,754 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:28,755 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 for the last time to pos 0, reset compression=false 2024-11-12T03:23:28,756 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 2024-11-12T03:23:28,760 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 was moved to hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 2024-11-12T03:23:28,760 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 and length of the file is 85 2024-11-12T03:23:28,760 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808609 2024-11-12T03:23:28,760 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808624, startPosition=0, beingWritten=true 2024-11-12T03:23:28,765 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731381808624: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:28,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741878_1054 (size=212) 2024-11-12T03:23:28,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741878_1054 (size=212) 2024-11-12T03:23:28,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741878_1054 (size=212) 2024-11-12T03:23:28,772 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:28,772 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381808624) 2024-11-12T03:23:28,780 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[1: isCompressionEnabled=true] Thread=355 (was 350) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #65] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57202 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #52] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=6418 (was 6426) 2024-11-12T03:23:28,789 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[1: isCompressionEnabled=true] Thread=355, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=449, ProcessCount=11, AvailableMemoryMB=6418 2024-11-12T03:23:28,789 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:28,792 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:28,793 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381808793 2024-11-12T03:23:28,799 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381808793 2024-11-12T03:23:28,800 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813)] 2024-11-12T03:23:28,803 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381808793, startPosition=0, beingWritten=true 2024-11-12T03:23:28,807 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381808793: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:28,810 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-12T03:23:28,811 DEBUG [Thread-717 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381808793, startPosition=0, beingWritten=true 2024-11-12T03:23:28,815 DEBUG [Thread-717 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381808793: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:28,817 DEBUG [Thread-717 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-12T03:23:28,817 DEBUG [Thread-717 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381808793 to pos 314, reset compression=false 2024-11-12T03:23:28,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741880_1056 (size=322) 2024-11-12T03:23:28,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741880_1056 (size=322) 2024-11-12T03:23:28,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741880_1056 (size=322) 2024-11-12T03:23:28,823 DEBUG [Thread-717 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731381808793 2024-11-12T03:23:28,823 DEBUG [Thread-717 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-12T03:23:28,823 DEBUG [Thread-717 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-12T03:23:28,825 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:28,825 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381808793) 2024-11-12T03:23:28,835 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[1: isCompressionEnabled=true] Thread=361 (was 355) Potentially hanging thread: Thread-717 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:343) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.replicationDone(ReplicationSourceWALReader.java:123) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:151) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57202 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #53] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 449), ProcessCount=11 (was 11), AvailableMemoryMB=6411 (was 6418) 2024-11-12T03:23:28,844 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[1: isCompressionEnabled=true] Thread=361, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=449, ProcessCount=11, AvailableMemoryMB=6410 2024-11-12T03:23:28,844 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:28,847 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEmptyStream_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:28,848 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731381808848 2024-11-12T03:23:28,857 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731381808848 2024-11-12T03:23:28,889 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:28,900 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731381808848, startPosition=0, beingWritten=true 2024-11-12T03:23:28,906 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731381808848: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:28,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:28,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741881_1057 (size=93) 2024-11-12T03:23:28,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741881_1057 (size=93) 2024-11-12T03:23:28,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741881_1057 (size=93) 2024-11-12T03:23:28,919 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:28,919 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381808848) 2024-11-12T03:23:28,931 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[1: isCompressionEnabled=true] Thread=368 (was 361) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57202 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 449), ProcessCount=11 (was 11), AvailableMemoryMB=6366 (was 6410) 2024-11-12T03:23:28,943 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[1: isCompressionEnabled=true] Thread=368, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=449, ProcessCount=11, AvailableMemoryMB=6366 2024-11-12T03:23:28,944 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:28,947 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:28,948 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381808947 2024-11-12T03:23:28,971 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381808947 2024-11-12T03:23:28,973 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:39813:39813)] 2024-11-12T03:23:28,979 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381808947, startPosition=0, beingWritten=true 2024-11-12T03:23:28,993 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381808947: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:28,999 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:29,000 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381808947, startPosition=204, beingWritten=true 2024-11-12T03:23:29,013 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381808947: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:29,017 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381809016 2024-11-12T03:23:29,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,026 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381808947 with entries=3, filesize=330 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381809016 2024-11-12T03:23:29,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741882_1058 (size=338) 2024-11-12T03:23:29,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741882_1058 (size=338) 2024-11-12T03:23:29,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741882_1058 (size=338) 2024-11-12T03:23:29,033 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985)] 2024-11-12T03:23:29,035 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:29,035 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381808947, startPosition=267, beingWritten=false 2024-11-12T03:23:29,038 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381808947: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:29,040 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:29,040 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381808947 2024-11-12T03:23:29,141 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381809016, startPosition=0, beingWritten=true 2024-11-12T03:23:29,145 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731381809016: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:29,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741883_1059 (size=212) 2024-11-12T03:23:29,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741883_1059 (size=212) 2024-11-12T03:23:29,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741883_1059 (size=212) 2024-11-12T03:23:29,154 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:29,154 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381809016) 2024-11-12T03:23:29,165 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[1: isCompressionEnabled=true] Thread=373 (was 368) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:41352 [Waiting for operation #69] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57202 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 449), ProcessCount=11 (was 11), AvailableMemoryMB=6360 (was 6366) 2024-11-12T03:23:29,178 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[1: isCompressionEnabled=true] Thread=373, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=449, ProcessCount=11, AvailableMemoryMB=6360 2024-11-12T03:23:29,179 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:29,183 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:29,184 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381809184 2024-11-12T03:23:29,198 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381809184 2024-11-12T03:23:29,199 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:29,210 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381809184, startPosition=0, beingWritten=true 2024-11-12T03:23:29,217 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381809184: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:29,218 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381809184, startPosition=204, beingWritten=true 2024-11-12T03:23:29,225 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731381809184: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:29,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741884_1060 (size=322) 2024-11-12T03:23:29,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741884_1060 (size=322) 2024-11-12T03:23:29,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741884_1060 (size=322) 2024-11-12T03:23:29,244 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:29,244 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381809184) 2024-11-12T03:23:29,255 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[1: isCompressionEnabled=true] Thread=378 (was 373) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:56502 [Waiting for operation #55] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:53290 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 449), ProcessCount=11 (was 11), AvailableMemoryMB=6359 (was 6360) 2024-11-12T03:23:29,264 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[1: isCompressionEnabled=true] Thread=378, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=449, ProcessCount=11, AvailableMemoryMB=6359 2024-11-12T03:23:29,264 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-12T03:23:29,267 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs, maxLogs=32 2024-11-12T03:23:29,268 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731381809267 2024-11-12T03:23:29,281 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731381809267 2024-11-12T03:23:29,297 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42817:42817),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:39813:39813)] 2024-11-12T03:23:29,304 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731381809304 2024-11-12T03:23:29,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,316 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731381809267 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731381809304 2024-11-12T03:23:29,321 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39813:39813),(127.0.0.1/127.0.0.1:36985:36985),(127.0.0.1/127.0.0.1:42817:42817)] 2024-11-12T03:23:29,321 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731381809267 is not closed yet, will try archiving it next time 2024-11-12T03:23:29,322 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T03:23:29,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741885_1061 (size=212) 2024-11-12T03:23:29,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741885_1061 (size=212) 2024-11-12T03:23:29,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741885_1061 (size=212) 2024-11-12T03:23:29,423 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731381809267, startPosition=0, beingWritten=false 2024-11-12T03:23:29,426 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731381809267: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-12T03:23:29,427 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38535/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731381809267 2024-11-12T03:23:29,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T03:23:29,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40231 is added to blk_1073741886_1062 (size=93) 2024-11-12T03:23:29,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741886_1062 (size=93) 2024-11-12T03:23:29,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46807 is added to blk_1073741886_1062 (size=93) 2024-11-12T03:23:29,434 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/oldWALs 2024-11-12T03:23:29,434 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D:(num 1731381809304) 2024-11-12T03:23:29,444 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[1: isCompressionEnabled=true] Thread=383 (was 378) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-638936435_22 at /127.0.0.1:57202 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=431 (was 431), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 449), ProcessCount=11 (was 11), AvailableMemoryMB=6354 (was 6359) 2024-11-12T03:23:29,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T03:23:29,488 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@55b5b1fa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T03:23:29,492 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2bdee3f9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T03:23:29,492 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T03:23:29,492 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32e0ec32{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T03:23:29,493 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1c21a271{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/hadoop.log.dir/,STOPPED} 2024-11-12T03:23:29,499 WARN [BP-1788124332-172.17.0.2-1731381795654 heartbeating to localhost/127.0.0.1:38535 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T03:23:29,499 WARN [BP-1788124332-172.17.0.2-1731381795654 heartbeating to localhost/127.0.0.1:38535 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1788124332-172.17.0.2-1731381795654 (Datanode Uuid 7c1c332d-7003-4dc6-8192-de0ecd333db6) service to localhost/127.0.0.1:38535 2024-11-12T03:23:29,502 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T03:23:29,502 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T03:23:29,502 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data5/current/BP-1788124332-172.17.0.2-1731381795654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T03:23:29,502 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data6/current/BP-1788124332-172.17.0.2-1731381795654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T03:23:29,503 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T03:23:29,512 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@71aaceb5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T03:23:29,512 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@39c5da84{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T03:23:29,513 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T03:23:29,513 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@212998b8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T03:23:29,513 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b6067f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/hadoop.log.dir/,STOPPED} 2024-11-12T03:23:29,515 WARN [BP-1788124332-172.17.0.2-1731381795654 heartbeating to localhost/127.0.0.1:38535 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T03:23:29,515 WARN [BP-1788124332-172.17.0.2-1731381795654 heartbeating to localhost/127.0.0.1:38535 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1788124332-172.17.0.2-1731381795654 (Datanode Uuid f9295c0b-a302-42a3-ab9e-7aef280a8e6e) service to localhost/127.0.0.1:38535 2024-11-12T03:23:29,517 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T03:23:29,517 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data4/current/BP-1788124332-172.17.0.2-1731381795654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T03:23:29,519 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T03:23:29,519 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T03:23:29,519 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data3/current/BP-1788124332-172.17.0.2-1731381795654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T03:23:29,529 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bc6d0f6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T03:23:29,530 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c1a1027{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T03:23:29,530 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T03:23:29,530 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54ac3606{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T03:23:29,531 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@229f9b5f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/hadoop.log.dir/,STOPPED} 2024-11-12T03:23:29,533 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T03:23:29,533 WARN [BP-1788124332-172.17.0.2-1731381795654 heartbeating to localhost/127.0.0.1:38535 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T03:23:29,534 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T03:23:29,534 WARN [BP-1788124332-172.17.0.2-1731381795654 heartbeating to localhost/127.0.0.1:38535 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1788124332-172.17.0.2-1731381795654 (Datanode Uuid 6596d77a-3a69-4811-8164-e228ce265e02) service to localhost/127.0.0.1:38535 2024-11-12T03:23:29,536 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data1/current/BP-1788124332-172.17.0.2-1731381795654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T03:23:29,537 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/cluster_a523b72d-21fb-88eb-fa7f-86b72b34b5f1/data/data2/current/BP-1788124332-172.17.0.2-1731381795654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T03:23:29,537 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T03:23:29,551 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1460f442{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T03:23:29,553 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@603bbd{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T03:23:29,553 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T03:23:29,553 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cf0e64{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T03:23:29,554 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47b3659{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bab5a210-9fea-d49c-ade7-21e4f03810d9/hadoop.log.dir/,STOPPED} 2024-11-12T03:23:29,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down