2024-11-20 21:21:39,655 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@69504ae9 2024-11-20 21:21:39,675 main DEBUG Took 0.016306 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 21:21:39,676 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 21:21:39,676 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 21:21:39,677 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 21:21:39,679 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:21:39,690 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 21:21:39,709 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:21:39,712 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,713 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:21:39,713 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,714 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:21:39,715 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,716 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:21:39,716 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,717 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:21:39,718 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,719 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:21:39,719 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:21:39,721 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,721 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:21:39,722 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,722 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:21:39,723 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,723 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:21:39,724 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,725 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:21:39,725 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,726 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:21:39,726 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,727 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 21:21:39,729 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:21:39,731 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 21:21:39,734 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 21:21:39,734 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 21:21:39,736 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 21:21:39,737 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 21:21:39,747 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 21:21:39,750 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 21:21:39,752 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 21:21:39,752 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 21:21:39,753 main DEBUG createAppenders(={Console}) 2024-11-20 21:21:39,754 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@69504ae9 initialized 2024-11-20 21:21:39,754 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@69504ae9 2024-11-20 21:21:39,754 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@69504ae9 OK. 2024-11-20 21:21:39,755 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 21:21:39,755 main DEBUG OutputStream closed 2024-11-20 21:21:39,755 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 21:21:39,756 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 21:21:39,756 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4659191b OK 2024-11-20 21:21:39,854 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 21:21:39,856 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 21:21:39,859 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 21:21:39,860 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 21:21:39,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 21:21:39,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 21:21:39,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 21:21:39,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 21:21:39,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 21:21:39,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 21:21:39,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 21:21:39,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 21:21:39,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 21:21:39,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 21:21:39,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 21:21:39,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 21:21:39,869 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 21:21:39,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 21:21:39,873 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 21:21:39,873 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@a87f8ec) with optional ClassLoader: null 2024-11-20 21:21:39,873 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 21:21:39,874 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@a87f8ec] started OK. 2024-11-20T21:21:40,311 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355 2024-11-20 21:21:40,315 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 21:21:40,315 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T21:21:40,364 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStreamFSHLog timeout: 26 mins 2024-11-20T21:21:40,407 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/cluster_284eea24-a7a8-0acf-c040-e490a965681e, deleteOnExit=true 2024-11-20T21:21:40,412 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/test.cache.data in system properties and HBase conf 2024-11-20T21:21:40,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T21:21:40,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/hadoop.log.dir in system properties and HBase conf 2024-11-20T21:21:40,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T21:21:40,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T21:21:40,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T21:21:40,714 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T21:21:40,869 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:21:40,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T21:21:40,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T21:21:40,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T21:21:40,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T21:21:40,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T21:21:40,881 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T21:21:40,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T21:21:40,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T21:21:40,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T21:21:40,885 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/nfs.dump.dir in system properties and HBase conf 2024-11-20T21:21:40,886 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/java.io.tmpdir in system properties and HBase conf 2024-11-20T21:21:40,886 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T21:21:40,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T21:21:40,888 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T21:21:42,220 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T21:21:42,315 INFO [Time-limited test {}] log.Log(170): Logging initialized @3856ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T21:21:42,421 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T21:21:42,508 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T21:21:42,544 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T21:21:42,545 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T21:21:42,548 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T21:21:42,569 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T21:21:42,574 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c2d1933{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/hadoop.log.dir/,AVAILABLE} 2024-11-20T21:21:42,577 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a9b22e2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T21:21:42,874 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@44567ea4{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/java.io.tmpdir/jetty-localhost-46713-hadoop-hdfs-3_4_1-tests_jar-_-any-14369035973579098809/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T21:21:42,900 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@216da5e5{HTTP/1.1, (http/1.1)}{localhost:46713} 2024-11-20T21:21:42,901 INFO [Time-limited test {}] server.Server(415): Started @4443ms 2024-11-20T21:21:43,456 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T21:21:43,467 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T21:21:43,471 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T21:21:43,472 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T21:21:43,472 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T21:21:43,476 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@309d3e97{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/hadoop.log.dir/,AVAILABLE} 2024-11-20T21:21:43,477 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d2a62e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T21:21:43,631 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5a86aa33{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/java.io.tmpdir/jetty-localhost-35047-hadoop-hdfs-3_4_1-tests_jar-_-any-14093020809688678583/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:21:43,633 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@53cf346{HTTP/1.1, (http/1.1)}{localhost:35047} 2024-11-20T21:21:43,633 INFO [Time-limited test {}] server.Server(415): Started @5176ms 2024-11-20T21:21:43,707 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T21:21:43,920 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T21:21:43,927 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T21:21:43,949 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T21:21:43,949 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T21:21:43,950 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T21:21:43,959 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cd634bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/hadoop.log.dir/,AVAILABLE} 2024-11-20T21:21:43,961 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45d510a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T21:21:44,185 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@fecc8a3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/java.io.tmpdir/jetty-localhost-33479-hadoop-hdfs-3_4_1-tests_jar-_-any-7570915868066443809/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:21:44,219 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@cbbb804{HTTP/1.1, (http/1.1)}{localhost:33479} 2024-11-20T21:21:44,224 INFO [Time-limited test {}] server.Server(415): Started @5761ms 2024-11-20T21:21:44,228 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T21:21:44,372 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T21:21:44,392 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T21:21:44,395 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T21:21:44,395 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T21:21:44,396 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T21:21:44,398 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f179af2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/hadoop.log.dir/,AVAILABLE} 2024-11-20T21:21:44,399 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dfa9f68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T21:21:44,499 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/cluster_284eea24-a7a8-0acf-c040-e490a965681e/data/data3/current/BP-1136921265-172.17.0.3-1732137701842/current, will proceed with Du for space computation calculation, 2024-11-20T21:21:44,502 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/cluster_284eea24-a7a8-0acf-c040-e490a965681e/data/data4/current/BP-1136921265-172.17.0.3-1732137701842/current, will proceed with Du for space computation calculation, 2024-11-20T21:21:44,505 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/cluster_284eea24-a7a8-0acf-c040-e490a965681e/data/data1/current/BP-1136921265-172.17.0.3-1732137701842/current, will proceed with Du for space computation calculation, 2024-11-20T21:21:44,508 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/cluster_284eea24-a7a8-0acf-c040-e490a965681e/data/data2/current/BP-1136921265-172.17.0.3-1732137701842/current, will proceed with Du for space computation calculation, 2024-11-20T21:21:44,595 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6d954ade{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/java.io.tmpdir/jetty-localhost-45987-hadoop-hdfs-3_4_1-tests_jar-_-any-2565602860654683526/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:21:44,596 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@76890781{HTTP/1.1, (http/1.1)}{localhost:45987} 2024-11-20T21:21:44,596 INFO [Time-limited test {}] server.Server(415): Started @6139ms 2024-11-20T21:21:44,603 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T21:21:44,629 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T21:21:44,633 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T21:21:44,757 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcd33d31c96012cee with lease ID 0x3524bfb5ac57a9a9: Processing first storage report for DS-3ae5767c-192b-4035-a082-0aa321718ebf from datanode DatanodeRegistration(127.0.0.1:37193, datanodeUuid=766fce34-fd71-4640-9e9d-b2afa05b747b, infoPort=41563, infoSecurePort=0, ipcPort=39029, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842) 2024-11-20T21:21:44,758 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcd33d31c96012cee with lease ID 0x3524bfb5ac57a9a9: from storage DS-3ae5767c-192b-4035-a082-0aa321718ebf node DatanodeRegistration(127.0.0.1:37193, datanodeUuid=766fce34-fd71-4640-9e9d-b2afa05b747b, infoPort=41563, infoSecurePort=0, ipcPort=39029, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-20T21:21:44,759 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcd33d31c96012cee with lease ID 0x3524bfb5ac57a9a9: Processing first storage report for DS-37dca507-7f83-4572-ba3d-a34c9d7e564e from datanode DatanodeRegistration(127.0.0.1:37193, datanodeUuid=766fce34-fd71-4640-9e9d-b2afa05b747b, infoPort=41563, infoSecurePort=0, ipcPort=39029, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842) 2024-11-20T21:21:44,759 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcd33d31c96012cee with lease ID 0x3524bfb5ac57a9a9: from storage DS-37dca507-7f83-4572-ba3d-a34c9d7e564e node DatanodeRegistration(127.0.0.1:37193, datanodeUuid=766fce34-fd71-4640-9e9d-b2afa05b747b, infoPort=41563, infoSecurePort=0, ipcPort=39029, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T21:21:44,762 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3bee216b5880337f with lease ID 0x3524bfb5ac57a9a8: Processing first storage report for DS-79bfdfe8-3154-4c62-9af3-329db70a583a from datanode DatanodeRegistration(127.0.0.1:32873, datanodeUuid=8292258b-1538-4d48-b80d-4f0574e06a21, infoPort=35485, infoSecurePort=0, ipcPort=41015, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842) 2024-11-20T21:21:44,762 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3bee216b5880337f with lease ID 0x3524bfb5ac57a9a8: from storage DS-79bfdfe8-3154-4c62-9af3-329db70a583a node DatanodeRegistration(127.0.0.1:32873, datanodeUuid=8292258b-1538-4d48-b80d-4f0574e06a21, infoPort=35485, infoSecurePort=0, ipcPort=41015, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T21:21:44,762 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3bee216b5880337f with lease ID 0x3524bfb5ac57a9a8: Processing first storage report for DS-beddce65-0ea8-4d5d-87b2-67f9cec9a44d from datanode DatanodeRegistration(127.0.0.1:32873, datanodeUuid=8292258b-1538-4d48-b80d-4f0574e06a21, infoPort=35485, infoSecurePort=0, ipcPort=41015, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842) 2024-11-20T21:21:44,763 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3bee216b5880337f with lease ID 0x3524bfb5ac57a9a8: from storage DS-beddce65-0ea8-4d5d-87b2-67f9cec9a44d node DatanodeRegistration(127.0.0.1:32873, datanodeUuid=8292258b-1538-4d48-b80d-4f0574e06a21, infoPort=35485, infoSecurePort=0, ipcPort=41015, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T21:21:44,827 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/cluster_284eea24-a7a8-0acf-c040-e490a965681e/data/data5/current/BP-1136921265-172.17.0.3-1732137701842/current, will proceed with Du for space computation calculation, 2024-11-20T21:21:44,840 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/cluster_284eea24-a7a8-0acf-c040-e490a965681e/data/data6/current/BP-1136921265-172.17.0.3-1732137701842/current, will proceed with Du for space computation calculation, 2024-11-20T21:21:44,919 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T21:21:44,929 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x219586af85fe26fa with lease ID 0x3524bfb5ac57a9aa: Processing first storage report for DS-23297ce8-74d4-457f-a938-d39c245664f8 from datanode DatanodeRegistration(127.0.0.1:43731, datanodeUuid=ac7636a0-3b69-421b-8050-d02f66429448, infoPort=39097, infoSecurePort=0, ipcPort=36029, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842) 2024-11-20T21:21:44,929 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x219586af85fe26fa with lease ID 0x3524bfb5ac57a9aa: from storage DS-23297ce8-74d4-457f-a938-d39c245664f8 node DatanodeRegistration(127.0.0.1:43731, datanodeUuid=ac7636a0-3b69-421b-8050-d02f66429448, infoPort=39097, infoSecurePort=0, ipcPort=36029, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T21:21:44,930 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x219586af85fe26fa with lease ID 0x3524bfb5ac57a9aa: Processing first storage report for DS-2a1e3e78-72a0-4078-8b38-13658548ebef from datanode DatanodeRegistration(127.0.0.1:43731, datanodeUuid=ac7636a0-3b69-421b-8050-d02f66429448, infoPort=39097, infoSecurePort=0, ipcPort=36029, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842) 2024-11-20T21:21:44,930 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x219586af85fe26fa with lease ID 0x3524bfb5ac57a9aa: from storage DS-2a1e3e78-72a0-4078-8b38-13658548ebef node DatanodeRegistration(127.0.0.1:43731, datanodeUuid=ac7636a0-3b69-421b-8050-d02f66429448, infoPort=39097, infoSecurePort=0, ipcPort=36029, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T21:21:45,235 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355 2024-11-20T21:21:45,263 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[0: isCompressionEnabled=false] Thread=158, OpenFileDescriptor=391, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=1784 2024-11-20T21:21:45,269 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T21:21:46,046 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T21:21:46,066 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:46,182 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:46,218 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1732137706213 2024-11-20T21:21:46,471 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 140 ms, current pipeline: [] 2024-11-20T21:21:46,479 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1732137706213 2024-11-20T21:21:46,621 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:46,706 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1732137706213, startPosition=0, beingWritten=true 2024-11-20T21:21:46,932 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1732137706213 to pos 203, reset compression=false 2024-11-20T21:21:46,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:46,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:46,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:46,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:46,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:47,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741825_1001 (size=447) 2024-11-20T21:21:47,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741825_1001 (size=447) 2024-11-20T21:21:47,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741825_1001 (size=447) 2024-11-20T21:21:47,427 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:47,430 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137706213) 2024-11-20T21:21:47,442 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[0: isCompressionEnabled=false] Thread=169 (was 158) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@1985baf7 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:43273 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=411 (was 391) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=765 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=1523 (was 1784) 2024-11-20T21:21:47,454 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[0: isCompressionEnabled=false] Thread=169, OpenFileDescriptor=411, MaxFileDescriptor=1048576, SystemLoadAverage=765, ProcessCount=11, AvailableMemoryMB=1521 2024-11-20T21:21:47,456 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:47,465 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:47,468 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137707467 2024-11-20T21:21:47,490 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137707467 2024-11-20T21:21:47,497 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:47,509 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137707467, startPosition=0, beingWritten=true 2024-11-20T21:21:47,755 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-20T21:21:47,772 DEBUG [Thread-176 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137707467, startPosition=0, beingWritten=true 2024-11-20T21:21:47,831 WARN [Thread-176 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-20T21:21:47,860 DEBUG [Thread-176 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137707467, startPosition=0, beingWritten=true 2024-11-20T21:21:47,884 WARN [Thread-176 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-20T21:21:47,915 DEBUG [Thread-176 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137707467, startPosition=0, beingWritten=true 2024-11-20T21:21:47,929 WARN [Thread-176 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-20T21:21:47,970 DEBUG [Thread-176 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137707467, startPosition=0, beingWritten=true 2024-11-20T21:21:47,982 DEBUG [Thread-176 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-20T21:21:47,983 DEBUG [Thread-176 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137707467 to pos 460, reset compression=false 2024-11-20T21:21:47,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:47,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:47,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:47,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:47,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741826_1002 (size=468) 2024-11-20T21:21:48,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741826_1002 (size=468) 2024-11-20T21:21:48,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741826_1002 (size=468) 2024-11-20T21:21:48,008 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:48,009 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137707467) 2024-11-20T21:21:48,011 DEBUG [Thread-176 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137707467 to pos 460, reset compression=false 2024-11-20T21:21:48,019 INFO [Thread-176 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137707467 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137707467 2024-11-20T21:21:48,019 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[0: isCompressionEnabled=false] Thread=176 (was 169) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-176 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=417 (was 411) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=765 (was 765), ProcessCount=11 (was 11), AvailableMemoryMB=1478 (was 1521) 2024-11-20T21:21:48,026 DEBUG [Thread-176 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137707467 2024-11-20T21:21:48,027 DEBUG [Thread-176 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:48,027 DEBUG [Thread-176 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:48,033 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[0: isCompressionEnabled=false] Thread=176, OpenFileDescriptor=417, MaxFileDescriptor=1048576, SystemLoadAverage=765, ProcessCount=11, AvailableMemoryMB=1477 2024-11-20T21:21:48,034 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:48,040 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:48,043 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D.1732137708042 2024-11-20T21:21:48,072 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_ isCompressionEnabled=false]/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D.1732137708042 2024-11-20T21:21:48,079 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:48,184 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor log.1.false 2024-11-20T21:21:48,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741828_1004 (size=231) 2024-11-20T21:21:48,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741828_1004 (size=231) 2024-11-20T21:21:48,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741828_1004 (size=231) 2024-11-20T21:21:48,223 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-20T21:21:48,224 DEBUG [Thread-199 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/user/jenkins/log.1.false, startPosition=0, beingWritten=false 2024-11-20T21:21:48,233 DEBUG [Thread-199 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/user/jenkins/log.1.false 2024-11-20T21:21:48,234 DEBUG [Thread-199 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-20T21:21:48,234 DEBUG [Thread-199 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/user/jenkins/log.2.false, startPosition=0, beingWritten=false 2024-11-20T21:21:48,237 WARN [Thread-199 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:43273/user/jenkins/log.2.false, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-20T21:21:48,238 DEBUG [Thread-199 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/user/jenkins/log.2.false 2024-11-20T21:21:48,238 DEBUG [Thread-199 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:48,238 DEBUG [Thread-199 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:48,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741827_1003 (size=93) 2024-11-20T21:21:48,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741827_1003 (size=93) 2024-11-20T21:21:48,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741827_1003 (size=93) 2024-11-20T21:21:48,255 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:48,255 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137708042) 2024-11-20T21:21:48,265 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[0: isCompressionEnabled=false] Thread=181 (was 176) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 417) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=765 (was 765), ProcessCount=11 (was 11), AvailableMemoryMB=1618 (was 1477) - AvailableMemoryMB LEAK? - 2024-11-20T21:21:48,276 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[0: isCompressionEnabled=false] Thread=181, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=765, ProcessCount=11, AvailableMemoryMB=1615 2024-11-20T21:21:48,277 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:48,285 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:48,287 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1732137708287 2024-11-20T21:21:48,313 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1732137708287 2024-11-20T21:21:48,317 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:48,321 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1732137708321 2024-11-20T21:21:48,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,352 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1732137708287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1732137708321 2024-11-20T21:21:48,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741829_1005 (size=93) 2024-11-20T21:21:48,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741829_1005 (size=93) 2024-11-20T21:21:48,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741829_1005 (size=93) 2024-11-20T21:21:48,370 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:48,378 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1732137708287 to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1732137708287 2024-11-20T21:21:48,380 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:48,392 WARN [Time-limited test {}] wal.AbstractFSWALProvider(471): Cannot parse a server name from path=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1732137708287 java.lang.IllegalStateException: null at org.apache.hbase.thirdparty.com.google.common.base.Preconditions.checkState(Preconditions.java:497) ~[hbase-shaded-miscellaneous-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.common.net.HostAndPort.getPort(HostAndPort.java:111) ~[hbase-shaded-miscellaneous-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.net.Address.getPort(Address.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ServerName.(ServerName.java:110) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ServerName.(ServerName.java:114) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ServerName.valueOf(ServerName.java:159) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ServerName.parseServerName(ServerName.java:330) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getServerNameFromWALDirectoryName(AbstractFSWALProvider.java:468) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.findArchivedLog(AbstractFSWALProvider.java:539) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testEOFExceptionInOldWALsDirectory(TestBasicWALEntryStream.java:781) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:21:48,395 WARN [Time-limited test {}] wal.AbstractFSWALProvider(541): Can not extract server name from path hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1732137708287, give up searching the separated old log dir 2024-11-20T21:21:48,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741830_1006 (size=93) 2024-11-20T21:21:48,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741830_1006 (size=93) 2024-11-20T21:21:48,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741830_1006 (size=93) 2024-11-20T21:21:48,419 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:48,419 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137708321) 2024-11-20T21:21:48,439 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[0: isCompressionEnabled=false] Thread=186 (was 181) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=765 (was 765), ProcessCount=11 (was 11), AvailableMemoryMB=1599 (was 1615) 2024-11-20T21:21:48,451 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[0: isCompressionEnabled=false] Thread=186, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=765, ProcessCount=11, AvailableMemoryMB=1596 2024-11-20T21:21:48,453 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:48,459 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:48,462 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1732137708461 2024-11-20T21:21:48,479 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1732137708461 2024-11-20T21:21:48,486 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:48,491 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1732137708491 2024-11-20T21:21:48,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,517 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1732137708461 with entries=10, filesize=1.30 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1732137708491 2024-11-20T21:21:48,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741831_1007 (size=1343) 2024-11-20T21:21:48,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741831_1007 (size=1343) 2024-11-20T21:21:48,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741831_1007 (size=1343) 2024-11-20T21:21:48,536 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:48,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741832_1008 (size=718) 2024-11-20T21:21:48,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741832_1008 (size=718) 2024-11-20T21:21:48,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741832_1008 (size=718) 2024-11-20T21:21:48,557 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=10, replicationBatchQueueCapacity=1 2024-11-20T21:21:48,559 DEBUG [Thread-232 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1732137708461, startPosition=0, beingWritten=false 2024-11-20T21:21:48,569 DEBUG [Thread-232 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-20T21:21:48,569 DEBUG [Thread-232 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1732137708461 2024-11-20T21:21:48,570 DEBUG [Thread-232 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1732137708491, startPosition=0, beingWritten=false 2024-11-20T21:21:48,582 DEBUG [Thread-232 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1732137708491 2024-11-20T21:21:48,582 DEBUG [Thread-232 {}] regionserver.ReplicationSourceWALReader(177): Read 5 WAL entries eligible for replication 2024-11-20T21:21:48,583 DEBUG [Thread-232 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:48,583 DEBUG [Thread-232 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:48,595 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:48,595 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137708491) 2024-11-20T21:21:48,610 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[0: isCompressionEnabled=false] Thread=191 (was 186) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=765 (was 765), ProcessCount=11 (was 11), AvailableMemoryMB=1575 (was 1596) 2024-11-20T21:21:48,622 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[0: isCompressionEnabled=false] Thread=191, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=765, ProcessCount=11, AvailableMemoryMB=1573 2024-11-20T21:21:48,623 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:48,629 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionForRecoveredQueue_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:48,632 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D.1732137708631 2024-11-20T21:21:48,652 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionForRecoveredQueue_0_ isCompressionEnabled=false]/testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D.1732137708631 2024-11-20T21:21:48,662 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:48,675 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-20T21:21:48,680 DEBUG [Thread-242 {}] regionserver.WALEntryStream(254): Creating new reader emptyLog, startPosition=0, beingWritten=false 2024-11-20T21:21:48,684 WARN [Thread-242 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: emptyLog, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-20T21:21:48,685 DEBUG [Thread-242 {}] regionserver.WALEntryStream(456): EOF, closing emptyLog 2024-11-20T21:21:48,685 DEBUG [Thread-242 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:48,685 DEBUG [Thread-242 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:48,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741833_1009 (size=93) 2024-11-20T21:21:48,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741833_1009 (size=93) 2024-11-20T21:21:48,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741833_1009 (size=93) 2024-11-20T21:21:48,713 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:48,713 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137708631) 2024-11-20T21:21:48,729 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[0: isCompressionEnabled=false] Thread=196 (was 191) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=765 (was 765), ProcessCount=11 (was 11), AvailableMemoryMB=1555 (was 1573) 2024-11-20T21:21:48,741 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[0: isCompressionEnabled=false] Thread=196, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=765, ProcessCount=11, AvailableMemoryMB=1553 2024-11-20T21:21:48,743 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:48,759 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:48,764 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708763 2024-11-20T21:21:48,841 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708763 2024-11-20T21:21:48,846 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:48,853 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708853 2024-11-20T21:21:48,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,885 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708763 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708853 2024-11-20T21:21:48,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741834_1010 (size=218) 2024-11-20T21:21:48,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741834_1010 (size=218) 2024-11-20T21:21:48,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741834_1010 (size=218) 2024-11-20T21:21:48,892 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:48,899 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:48,902 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-20T21:21:48,904 DEBUG [Thread-259 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708763, startPosition=0, beingWritten=false 2024-11-20T21:21:48,911 DEBUG [Thread-259 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708763 2024-11-20T21:21:48,911 DEBUG [Thread-259 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-20T21:21:48,912 DEBUG [Thread-259 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708853, startPosition=0, beingWritten=true 2024-11-20T21:21:48,929 DEBUG [Thread-259 {}] regionserver.ReplicationSourceWALReader(177): Read 20 WAL entries eligible for replication 2024-11-20T21:21:48,929 DEBUG [Thread-259 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708853 to pos 2585, reset compression=false 2024-11-20T21:21:48,929 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708929 2024-11-20T21:21:48,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,949 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708853 with entries=20, filesize=2.52 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708929 2024-11-20T21:21:48,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741835_1011 (size=2593) 2024-11-20T21:21:48,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741835_1011 (size=2593) 2024-11-20T21:21:48,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741835_1011 (size=2593) 2024-11-20T21:21:48,959 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:48,959 DEBUG [Thread-259 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708853 for the last time to pos 2585, reset compression=false 2024-11-20T21:21:48,965 DEBUG [Thread-259 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708853 and length of the file is 2593 2024-11-20T21:21:48,965 DEBUG [Thread-259 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708853 2024-11-20T21:21:48,966 DEBUG [Thread-259 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708929, startPosition=0, beingWritten=true 2024-11-20T21:21:48,982 DEBUG [Thread-259 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-20T21:21:48,983 DEBUG [Thread-259 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708929 to pos 1335, reset compression=false 2024-11-20T21:21:48,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:48,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741836_1012 (size=1343) 2024-11-20T21:21:48,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741836_1012 (size=1343) 2024-11-20T21:21:48,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741836_1012 (size=1343) 2024-11-20T21:21:49,000 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:49,001 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137708929) 2024-11-20T21:21:49,007 DEBUG [Thread-259 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137708929 2024-11-20T21:21:49,008 DEBUG [Thread-259 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:49,008 DEBUG [Thread-259 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:49,016 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[0: isCompressionEnabled=false] Thread=202 (was 196) Potentially hanging thread: Thread-259 app//org.apache.hadoop.thirdparty.protobuf.GeneratedMessageV3$Builder.(GeneratedMessageV3.java:587) app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$ReadOpChecksumInfoProto$Builder.(DataTransferProtos.java:25649) app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$ReadOpChecksumInfoProto$Builder.(DataTransferProtos.java:25625) app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$ReadOpChecksumInfoProto.newBuilderForType(DataTransferProtos.java:25613) app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$ReadOpChecksumInfoProto.newBuilderForType(DataTransferProtos.java:25333) app//org.apache.hadoop.thirdparty.protobuf.GeneratedMessageV3.newBuilderForType(GeneratedMessageV3.java:548) app//org.apache.hadoop.thirdparty.protobuf.SingleFieldBuilderV3.getBuilder(SingleFieldBuilderV3.java:108) app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$BlockOpResponseProto$Builder.mergeFrom(DataTransferProtos.java:26965) app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$BlockOpResponseProto$1.parsePartialFrom(DataTransferProtos.java:27575) app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$BlockOpResponseProto$1.parsePartialFrom(DataTransferProtos.java:27567) app//org.apache.hadoop.thirdparty.protobuf.AbstractParser.parsePartialFrom(AbstractParser.java:192) app//org.apache.hadoop.thirdparty.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) app//org.apache.hadoop.thirdparty.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) app//org.apache.hadoop.thirdparty.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) app//org.apache.hadoop.thirdparty.protobuf.GeneratedMessageV3.parseWithIOException(GeneratedMessageV3.java:332) app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$BlockOpResponseProto.parseFrom(DataTransferProtos.java:26657) app//org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:418) app//org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) app//org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) app//org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) app//org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) app//org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) app//org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) java.base@17.0.11/java.io.DataInputStream.readInt(DataInputStream.java:381) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:628) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=765 (was 765), ProcessCount=11 (was 11), AvailableMemoryMB=1526 (was 1553) 2024-11-20T21:21:49,027 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[0: isCompressionEnabled=false] Thread=202, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=765, ProcessCount=11, AvailableMemoryMB=1525 2024-11-20T21:21:49,028 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:49,034 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALKeySerialization_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:49,035 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D.1732137709035 2024-11-20T21:21:49,055 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALKeySerialization_0_ isCompressionEnabled=false]/testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D.1732137709035 2024-11-20T21:21:49,059 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:49,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741837_1013 (size=93) 2024-11-20T21:21:49,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741837_1013 (size=93) 2024-11-20T21:21:49,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741837_1013 (size=93) 2024-11-20T21:21:49,085 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:49,085 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137709035) 2024-11-20T21:21:49,103 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[0: isCompressionEnabled=false] Thread=207 (was 202) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=765 (was 765), ProcessCount=11 (was 11), AvailableMemoryMB=1522 (was 1525) 2024-11-20T21:21:49,114 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[0: isCompressionEnabled=false] Thread=207, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=765, ProcessCount=11, AvailableMemoryMB=1522 2024-11-20T21:21:49,115 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:49,121 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:49,122 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1732137709122 2024-11-20T21:21:49,142 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1732137709122 2024-11-20T21:21:49,148 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:49,156 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1732137709122, startPosition=0, beingWritten=true 2024-11-20T21:21:49,171 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-20T21:21:49,174 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:49,285 DEBUG [Thread-289 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1732137709122, startPosition=0, beingWritten=true 2024-11-20T21:21:49,297 DEBUG [Thread-289 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-20T21:21:49,297 DEBUG [Thread-289 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1732137709122 to pos 460, reset compression=false 2024-11-20T21:21:49,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741838_1014 (size=468) 2024-11-20T21:21:49,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741838_1014 (size=468) 2024-11-20T21:21:49,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741838_1014 (size=468) 2024-11-20T21:21:49,716 DEBUG [Thread-289 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1732137709122 for the last time to pos 460, reset compression=false 2024-11-20T21:21:49,721 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:49,722 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137709122) 2024-11-20T21:21:49,729 INFO [Thread-289 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1732137709122 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1732137709122 2024-11-20T21:21:49,730 DEBUG [Thread-289 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1732137709122 and length of the file is 468 2024-11-20T21:21:49,731 DEBUG [Thread-289 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1732137709122 2024-11-20T21:21:49,731 DEBUG [Thread-289 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:49,732 DEBUG [Thread-289 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:49,739 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[0: isCompressionEnabled=false] Thread=215 (was 207) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-289 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy48.getFileInfo(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getFileInfo$41(ClientNamenodeProtocolTranslatorPB.java:820) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$309/0x00007fbff87b2b50.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:820) jdk.internal.reflect.GeneratedMethodAccessor30.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy49.getFileInfo(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1770) app//org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1828) app//org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1825) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1840) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.getCurrentPathFileStatus(WALEntryStream.java:399) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.checkAllBytesParsed(WALEntryStream.java:417) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:315) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=765 (was 765), ProcessCount=11 (was 11), AvailableMemoryMB=1534 (was 1522) - AvailableMemoryMB LEAK? - 2024-11-20T21:21:49,751 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[0: isCompressionEnabled=false] Thread=215, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=765, ProcessCount=11, AvailableMemoryMB=1532 2024-11-20T21:21:49,752 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:49,772 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:49,775 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1732137709774 2024-11-20T21:21:49,822 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1732137709774 2024-11-20T21:21:49,831 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:49,857 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:49,857 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1732137709774, startPosition=0, beingWritten=true 2024-11-20T21:21:49,870 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1732137709870 2024-11-20T21:21:49,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:49,891 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1732137709774 with entries=3, filesize=439 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1732137709870 2024-11-20T21:21:49,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741839_1015 (size=447) 2024-11-20T21:21:49,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741839_1015 (size=447) 2024-11-20T21:21:49,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741839_1015 (size=447) 2024-11-20T21:21:49,901 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:49,903 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:49,903 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:49,903 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1732137709774 for the last time to pos 321, reset compression=false 2024-11-20T21:21:49,908 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:49,909 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1732137709774 2024-11-20T21:21:50,009 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1732137709870, startPosition=0, beingWritten=true 2024-11-20T21:21:50,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741840_1016 (size=211) 2024-11-20T21:21:50,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741840_1016 (size=211) 2024-11-20T21:21:50,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741840_1016 (size=211) 2024-11-20T21:21:50,030 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:50,030 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137709870) 2024-11-20T21:21:50,041 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[0: isCompressionEnabled=false] Thread=220 (was 215) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=765 (was 765), ProcessCount=11 (was 11), AvailableMemoryMB=1518 (was 1532) 2024-11-20T21:21:50,051 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[0: isCompressionEnabled=false] Thread=220, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=765, ProcessCount=11, AvailableMemoryMB=1518 2024-11-20T21:21:50,052 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:50,056 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:50,057 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1732137710057 2024-11-20T21:21:50,067 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1732137710057 2024-11-20T21:21:50,069 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:50,078 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:50,078 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1732137710057, startPosition=0, beingWritten=true 2024-11-20T21:21:50,087 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1732137710087 2024-11-20T21:21:50,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,100 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1732137710057 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1732137710087 2024-11-20T21:21:50,105 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:50,105 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1732137710057 is not closed yet, will try archiving it next time 2024-11-20T21:21:50,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741841_1017 (size=218) 2024-11-20T21:21:50,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741841_1017 (size=218) 2024-11-20T21:21:50,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741841_1017 (size=218) 2024-11-20T21:21:50,109 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:50,109 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1732137710057 for the last time to pos 210, reset compression=false 2024-11-20T21:21:50,113 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1732137710057 and length of the file is 218 2024-11-20T21:21:50,113 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1732137710057 2024-11-20T21:21:50,214 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1732137710087, startPosition=0, beingWritten=true 2024-11-20T21:21:50,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:50,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741842_1018 (size=218) 2024-11-20T21:21:50,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741842_1018 (size=218) 2024-11-20T21:21:50,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741842_1018 (size=218) 2024-11-20T21:21:50,241 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:50,241 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137710087) 2024-11-20T21:21:50,252 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[0: isCompressionEnabled=false] Thread=225 (was 220) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=765 (was 765), ProcessCount=11 (was 11), AvailableMemoryMB=1579 (was 1518) - AvailableMemoryMB LEAK? - 2024-11-20T21:21:50,261 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[0: isCompressionEnabled=false] Thread=225, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=765, ProcessCount=11, AvailableMemoryMB=1577 2024-11-20T21:21:50,262 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:50,269 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:50,270 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1732137710270 2024-11-20T21:21:50,292 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1732137710270 2024-11-20T21:21:50,308 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:50,326 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1732137710270, startPosition=0, beingWritten=true 2024-11-20T21:21:50,347 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 272 and read up to 320 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:612) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.KeyValueUtil.createKeyValueFromInputStream(KeyValueUtil.java:659) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.KeyValueCodecWithTags$KeyValueDecoder.parseCell(KeyValueCodecWithTags.java:77) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-20T21:21:51,348 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1732137710270 to pos 203, reset compression=false 2024-11-20T21:21:51,370 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 272 and read up to 320 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:615) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.KeyValueUtil.createKeyValueFromInputStream(KeyValueUtil.java:659) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.KeyValueCodecWithTags$KeyValueDecoder.parseCell(KeyValueCodecWithTags.java:77) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-20T21:21:51,371 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1732137710270 to pos 203, reset compression=false 2024-11-20T21:21:51,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741843_1019 (size=329) 2024-11-20T21:21:51,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741843_1019 (size=329) 2024-11-20T21:21:51,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741843_1019 (size=329) 2024-11-20T21:21:51,414 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:51,414 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137710270) 2024-11-20T21:21:51,426 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[0: isCompressionEnabled=false] Thread=229 (was 225) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 765), ProcessCount=11 (was 11), AvailableMemoryMB=1462 (was 1577) 2024-11-20T21:21:51,441 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[0: isCompressionEnabled=false] Thread=229, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1461 2024-11-20T21:21:51,442 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:51,448 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:51,449 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137711449 2024-11-20T21:21:51,483 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137711449 2024-11-20T21:21:51,493 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:51,501 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137711449, startPosition=0, beingWritten=true 2024-11-20T21:21:51,521 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137711449, startPosition=203, beingWritten=true 2024-11-20T21:21:51,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741844_1020 (size=447) 2024-11-20T21:21:51,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741844_1020 (size=447) 2024-11-20T21:21:51,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741844_1020 (size=447) 2024-11-20T21:21:51,545 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:51,545 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137711449) 2024-11-20T21:21:51,555 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[0: isCompressionEnabled=false] Thread=234 (was 229) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1455 (was 1461) 2024-11-20T21:21:51,560 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:21:51,566 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[0: isCompressionEnabled=false] Thread=234, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1455 2024-11-20T21:21:51,568 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:51,574 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:51,575 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1732137711575 2024-11-20T21:21:51,599 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1732137711575 2024-11-20T21:21:51,600 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:51,620 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1732137711575, startPosition=0, beingWritten=true 2024-11-20T21:21:51,632 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-20T21:21:51,633 DEBUG [Thread-359 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1732137711575, startPosition=0, beingWritten=true 2024-11-20T21:21:51,653 DEBUG [Thread-359 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-20T21:21:51,653 DEBUG [Thread-359 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1732137711575 to pos 460, reset compression=false 2024-11-20T21:21:51,666 DEBUG [Thread-359 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-20T21:21:51,666 DEBUG [Thread-359 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1732137711575 to pos 580, reset compression=false 2024-11-20T21:21:51,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741845_1021 (size=588) 2024-11-20T21:21:51,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741845_1021 (size=588) 2024-11-20T21:21:51,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741845_1021 (size=588) 2024-11-20T21:21:51,680 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:51,681 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137711575) 2024-11-20T21:21:51,691 INFO [Thread-359 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1732137711575 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1732137711575 2024-11-20T21:21:51,696 DEBUG [Thread-359 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1732137711575 2024-11-20T21:21:51,696 DEBUG [Thread-359 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:51,696 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[0: isCompressionEnabled=false] Thread=241 (was 234) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Thread-359 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.offer(SynchronousQueue.java:857) app//org.apache.hadoop.ipc.Client$Connection.sendRpcRequest(Client.java:1187) app//org.apache.hadoop.ipc.Client.call(Client.java:1479) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy85.getReplicaVisibleLength(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB.lambda$getReplicaVisibleLength$0(ClientDatanodeProtocolTranslatorPB.java:199) app//org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB$$Lambda$341/0x00007fbff8894e10.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB.getReplicaVisibleLength(ClientDatanodeProtocolTranslatorPB.java:199) app//org.apache.hadoop.hdfs.DFSInputStream.readBlockLength(DFSInputStream.java:352) app//org.apache.hadoop.hdfs.DFSInputStream.getLastBlockLength(DFSInputStream.java:323) app//org.apache.hadoop.hdfs.DFSInputStream.openInfo(DFSInputStream.java:243) app//org.apache.hadoop.hdfs.DFSInputStream.(DFSInputStream.java:213) app//org.apache.hadoop.hdfs.DFSClient.openInternal(DFSClient.java:1114) app//org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1077) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:342) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:338) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:355) app//org.apache.hadoop.fs.FileSystem.open(FileSystem.java:997) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1452 (was 1455) 2024-11-20T21:21:51,697 DEBUG [Thread-359 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:51,708 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[0: isCompressionEnabled=false] Thread=241, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1452 2024-11-20T21:21:51,709 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:51,714 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:51,714 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 2024-11-20T21:21:51,725 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 2024-11-20T21:21:51,733 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:51,737 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711737 2024-11-20T21:21:51,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,748 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711737 2024-11-20T21:21:51,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741846_1022 (size=93) 2024-11-20T21:21:51,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741846_1022 (size=93) 2024-11-20T21:21:51,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741846_1022 (size=93) 2024-11-20T21:21:51,757 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:51,759 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 2024-11-20T21:21:51,760 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:51,763 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 2024-11-20T21:21:51,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741848_1024 (size=85) 2024-11-20T21:21:51,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741848_1024 (size=85) 2024-11-20T21:21:51,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741848_1024 (size=85) 2024-11-20T21:21:51,789 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714, startPosition=0, beingWritten=false 2024-11-20T21:21:51,791 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 2024-11-20T21:21:51,795 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 for the last time to pos 0, reset compression=false 2024-11-20T21:21:51,797 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 2024-11-20T21:21:51,802 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 2024-11-20T21:21:51,803 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 and length of the file is 85 2024-11-20T21:21:51,803 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711714 2024-11-20T21:21:51,803 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1732137711737, startPosition=0, beingWritten=true 2024-11-20T21:21:51,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:51,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741847_1023 (size=218) 2024-11-20T21:21:51,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741847_1023 (size=218) 2024-11-20T21:21:51,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741847_1023 (size=218) 2024-11-20T21:21:51,821 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:51,821 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137711737) 2024-11-20T21:21:51,831 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[0: isCompressionEnabled=false] Thread=246 (was 241) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1449 (was 1452) 2024-11-20T21:21:51,842 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[0: isCompressionEnabled=false] Thread=246, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1448 2024-11-20T21:21:51,844 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:51,849 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:51,850 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137711850 2024-11-20T21:21:51,862 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137711850 2024-11-20T21:21:51,868 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:51,873 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137711850, startPosition=0, beingWritten=true 2024-11-20T21:21:51,884 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-20T21:21:51,885 DEBUG [Thread-397 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137711850, startPosition=0, beingWritten=true 2024-11-20T21:21:51,893 WARN [Thread-397 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-20T21:21:51,914 DEBUG [Thread-397 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137711850, startPosition=0, beingWritten=true 2024-11-20T21:21:51,922 WARN [Thread-397 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-20T21:21:51,953 DEBUG [Thread-397 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137711850, startPosition=0, beingWritten=true 2024-11-20T21:21:51,961 WARN [Thread-397 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-20T21:21:52,002 DEBUG [Thread-397 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137711850, startPosition=0, beingWritten=true 2024-11-20T21:21:52,031 WARN [Thread-397 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-20T21:21:52,082 DEBUG [Thread-397 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137711850, startPosition=0, beingWritten=true 2024-11-20T21:21:52,092 WARN [Thread-397 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-20T21:21:52,153 DEBUG [Thread-397 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137711850, startPosition=0, beingWritten=true 2024-11-20T21:21:52,182 DEBUG [Thread-397 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-20T21:21:52,182 DEBUG [Thread-397 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137711850 to pos 460, reset compression=false 2024-11-20T21:21:52,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741849_1025 (size=468) 2024-11-20T21:21:52,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741849_1025 (size=468) 2024-11-20T21:21:52,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741849_1025 (size=468) 2024-11-20T21:21:52,222 DEBUG [Thread-397 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1732137711850 2024-11-20T21:21:52,222 DEBUG [Thread-397 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:52,222 DEBUG [Thread-397 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:52,227 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:52,227 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137711850) 2024-11-20T21:21:52,241 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[0: isCompressionEnabled=false] Thread=252 (was 246) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-397 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:343) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.replicationDone(ReplicationSourceWALReader.java:123) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:151) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1564 (was 1448) - AvailableMemoryMB LEAK? - 2024-11-20T21:21:52,265 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[0: isCompressionEnabled=false] Thread=252, OpenFileDescriptor=429, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1561 2024-11-20T21:21:52,266 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:52,293 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEmptyStream_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:52,294 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1732137712294 2024-11-20T21:21:52,317 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEmptyStream_0_ isCompressionEnabled=false]/testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1732137712294 2024-11-20T21:21:52,322 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:52,327 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEmptyStream_0_ isCompressionEnabled=false]/testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1732137712294, startPosition=0, beingWritten=true 2024-11-20T21:21:52,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741850_1026 (size=93) 2024-11-20T21:21:52,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741850_1026 (size=93) 2024-11-20T21:21:52,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741850_1026 (size=93) 2024-11-20T21:21:52,347 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:52,347 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137712294) 2024-11-20T21:21:52,360 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[0: isCompressionEnabled=false] Thread=257 (was 252) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 429), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1552 (was 1561) 2024-11-20T21:21:52,375 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[0: isCompressionEnabled=false] Thread=257, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1549 2024-11-20T21:21:52,377 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:52,385 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:52,386 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1732137712386 2024-11-20T21:21:52,397 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1732137712386 2024-11-20T21:21:52,406 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:52,411 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1732137712386, startPosition=0, beingWritten=true 2024-11-20T21:21:52,428 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:52,429 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1732137712386, startPosition=210, beingWritten=true 2024-11-20T21:21:52,451 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1732137712451 2024-11-20T21:21:52,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,478 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1732137712386 with entries=3, filesize=460 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1732137712451 2024-11-20T21:21:52,481 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:52,481 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1732137712386 is not closed yet, will try archiving it next time 2024-11-20T21:21:52,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741851_1027 (size=468) 2024-11-20T21:21:52,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741851_1027 (size=468) 2024-11-20T21:21:52,484 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:52,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741851_1027 (size=468) 2024-11-20T21:21:52,484 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1732137712386, startPosition=335, beingWritten=true 2024-11-20T21:21:52,489 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:52,489 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1732137712386 2024-11-20T21:21:52,590 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1732137712451, startPosition=0, beingWritten=true 2024-11-20T21:21:52,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741852_1028 (size=218) 2024-11-20T21:21:52,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741852_1028 (size=218) 2024-11-20T21:21:52,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741852_1028 (size=218) 2024-11-20T21:21:52,670 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:52,670 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137712451) 2024-11-20T21:21:52,685 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[0: isCompressionEnabled=false] Thread=262 (was 257) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1503 (was 1549) 2024-11-20T21:21:52,700 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[0: isCompressionEnabled=false] Thread=262, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1502 2024-11-20T21:21:52,701 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:52,708 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:52,710 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137712709 2024-11-20T21:21:52,749 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137712709 2024-11-20T21:21:52,757 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:52,764 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137712709, startPosition=0, beingWritten=true 2024-11-20T21:21:52,783 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1732137712709, startPosition=210, beingWritten=true 2024-11-20T21:21:52,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741853_1029 (size=468) 2024-11-20T21:21:52,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741853_1029 (size=468) 2024-11-20T21:21:52,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741853_1029 (size=468) 2024-11-20T21:21:52,811 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:52,811 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137712709) 2024-11-20T21:21:52,827 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[0: isCompressionEnabled=false] Thread=267 (was 262) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1500 (was 1502) 2024-11-20T21:21:52,841 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[0: isCompressionEnabled=false] Thread=267, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1499 2024-11-20T21:21:52,842 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:52,847 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:52,849 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1732137712848 2024-11-20T21:21:52,865 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1732137712848 2024-11-20T21:21:52,869 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:52,871 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1732137712871 2024-11-20T21:21:52,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,903 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1732137712848 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1732137712871 2024-11-20T21:21:52,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741854_1030 (size=218) 2024-11-20T21:21:52,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741854_1030 (size=218) 2024-11-20T21:21:52,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741854_1030 (size=218) 2024-11-20T21:21:52,909 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:52,909 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:52,913 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1732137712848, startPosition=0, beingWritten=false 2024-11-20T21:21:52,917 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1732137712848 2024-11-20T21:21:52,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:52,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741855_1031 (size=93) 2024-11-20T21:21:52,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741855_1031 (size=93) 2024-11-20T21:21:52,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741855_1031 (size=93) 2024-11-20T21:21:52,927 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:52,927 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D:(num 1732137712871) 2024-11-20T21:21:52,938 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[0: isCompressionEnabled=false] Thread=272 (was 267) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1496 (was 1499) 2024-11-20T21:21:52,951 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[1: isCompressionEnabled=true] Thread=272, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1496 2024-11-20T21:21:52,952 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:52,956 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:52,957 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137712957 2024-11-20T21:21:52,969 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137712957 2024-11-20T21:21:52,975 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:52,978 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137712957, startPosition=0, beingWritten=true 2024-11-20T21:21:52,987 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137712957: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:52,993 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137712957 to pos 197, reset compression=false 2024-11-20T21:21:53,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741856_1032 (size=317) 2024-11-20T21:21:53,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741856_1032 (size=317) 2024-11-20T21:21:53,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741856_1032 (size=317) 2024-11-20T21:21:53,009 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:53,009 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137712957) 2024-11-20T21:21:53,020 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[1: isCompressionEnabled=true] Thread=277 (was 272) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #49] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1491 (was 1496) 2024-11-20T21:21:53,031 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[1: isCompressionEnabled=true] Thread=277, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1490 2024-11-20T21:21:53,031 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:53,035 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:53,036 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036 2024-11-20T21:21:53,045 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036 2024-11-20T21:21:53,049 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:53,053 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036, startPosition=0, beingWritten=true 2024-11-20T21:21:53,063 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:53,067 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-20T21:21:53,068 DEBUG [Thread-478 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036, startPosition=0, beingWritten=true 2024-11-20T21:21:53,075 DEBUG [Thread-478 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:53,078 WARN [Thread-478 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-20T21:21:53,098 DEBUG [Thread-478 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036, startPosition=0, beingWritten=true 2024-11-20T21:21:53,105 DEBUG [Thread-478 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:53,108 WARN [Thread-478 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-20T21:21:53,139 DEBUG [Thread-478 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036, startPosition=0, beingWritten=true 2024-11-20T21:21:53,146 DEBUG [Thread-478 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:53,150 WARN [Thread-478 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-20T21:21:53,191 DEBUG [Thread-478 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036, startPosition=0, beingWritten=true 2024-11-20T21:21:53,199 DEBUG [Thread-478 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:53,203 DEBUG [Thread-478 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-20T21:21:53,203 DEBUG [Thread-478 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036 to pos 314, reset compression=false 2024-11-20T21:21:53,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741857_1033 (size=322) 2024-11-20T21:21:53,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741857_1033 (size=322) 2024-11-20T21:21:53,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741857_1033 (size=322) 2024-11-20T21:21:53,239 DEBUG [Thread-478 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036 for the last time to pos 314, reset compression=false 2024-11-20T21:21:53,246 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:53,246 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137713036) 2024-11-20T21:21:53,252 INFO [Thread-478 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036 2024-11-20T21:21:53,262 INFO [Thread-478 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036 2024-11-20T21:21:53,263 DEBUG [Thread-478 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036 and length of the file is 322 2024-11-20T21:21:53,263 DEBUG [Thread-478 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137713036 2024-11-20T21:21:53,264 DEBUG [Thread-478 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:53,264 DEBUG [Thread-478 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:53,269 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[1: isCompressionEnabled=true] Thread=283 (was 277) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-478 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy48.getFileInfo(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getFileInfo$41(ClientNamenodeProtocolTranslatorPB.java:820) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$309/0x00007fbff87b2b50.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:820) jdk.internal.reflect.GeneratedMethodAccessor30.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy49.getFileInfo(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1770) app//org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1828) app//org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1825) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1840) app//org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1862) app//org.apache.hadoop.hbase.wal.AbstractFSWALProvider.findArchivedLog(AbstractFSWALProvider.java:534) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.openArchivedWAL(AbstractProtobufWALReader.java:183) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:196) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #56] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1534 (was 1490) - AvailableMemoryMB LEAK? - 2024-11-20T21:21:53,282 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[1: isCompressionEnabled=true] Thread=283, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1533 2024-11-20T21:21:53,283 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:53,297 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:53,299 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D.1732137713299 2024-11-20T21:21:53,331 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_ isCompressionEnabled=true]/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D.1732137713299 2024-11-20T21:21:53,346 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:53,353 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor log.1.true 2024-11-20T21:21:53,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741859_1035 (size=225) 2024-11-20T21:21:53,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741859_1035 (size=225) 2024-11-20T21:21:53,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741859_1035 (size=225) 2024-11-20T21:21:53,382 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-20T21:21:53,385 DEBUG [Thread-501 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/user/jenkins/log.1.true, startPosition=0, beingWritten=false 2024-11-20T21:21:53,392 DEBUG [Thread-501 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/user/jenkins/log.1.true: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:53,395 DEBUG [Thread-501 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/user/jenkins/log.1.true 2024-11-20T21:21:53,395 DEBUG [Thread-501 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-20T21:21:53,396 DEBUG [Thread-501 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/user/jenkins/log.2.true, startPosition=0, beingWritten=false 2024-11-20T21:21:53,397 WARN [Thread-501 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:43273/user/jenkins/log.2.true, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-20T21:21:53,398 DEBUG [Thread-501 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/user/jenkins/log.2.true 2024-11-20T21:21:53,398 DEBUG [Thread-501 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:53,398 DEBUG [Thread-501 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:53,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741858_1034 (size=93) 2024-11-20T21:21:53,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741858_1034 (size=93) 2024-11-20T21:21:53,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741858_1034 (size=93) 2024-11-20T21:21:53,407 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:53,407 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137713299) 2024-11-20T21:21:53,421 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[1: isCompressionEnabled=true] Thread=288 (was 283) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #59] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1511 (was 1533) 2024-11-20T21:21:53,431 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[1: isCompressionEnabled=true] Thread=288, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1510 2024-11-20T21:21:53,432 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:53,436 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:53,437 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713436 2024-11-20T21:21:53,450 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713436 2024-11-20T21:21:53,457 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:53,478 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713477 2024-11-20T21:21:53,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,497 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713477 2024-11-20T21:21:53,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741860_1036 (size=93) 2024-11-20T21:21:53,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741860_1036 (size=93) 2024-11-20T21:21:53,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741860_1036 (size=93) 2024-11-20T21:21:53,513 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713436 to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713436 2024-11-20T21:21:53,517 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:53,517 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:53,519 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713436 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713436 2024-11-20T21:21:53,533 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-20T21:21:53,537 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:53,538 DEBUG [Thread-518 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713436, startPosition=0, beingWritten=false 2024-11-20T21:21:53,540 INFO [Thread-518 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713436 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713436 2024-11-20T21:21:53,542 WARN [Thread-518 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713436, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-20T21:21:53,542 DEBUG [Thread-518 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713436 2024-11-20T21:21:53,542 DEBUG [Thread-518 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713477, startPosition=0, beingWritten=true 2024-11-20T21:21:53,557 DEBUG [Thread-518 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713477: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:53,569 DEBUG [Thread-518 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713477 to pos 0, reset compression=false 2024-11-20T21:21:53,593 DEBUG [Thread-518 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713477 to pos 0, reset compression=false 2024-11-20T21:21:53,647 DEBUG [Thread-518 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713477 to pos 0, reset compression=false 2024-11-20T21:21:53,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741861_1037 (size=93) 2024-11-20T21:21:53,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741861_1037 (size=93) 2024-11-20T21:21:53,707 DEBUG [Thread-518 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1732137713477 2024-11-20T21:21:53,708 DEBUG [Thread-518 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:53,708 DEBUG [Thread-518 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:53,708 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:53,708 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137713477) 2024-11-20T21:21:53,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741861_1037 (size=93) 2024-11-20T21:21:53,747 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[1: isCompressionEnabled=true] Thread=294 (was 288) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #61] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-518 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:343) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.replicationDone(ReplicationSourceWALReader.java:123) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:151) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1458 (was 1510) 2024-11-20T21:21:53,762 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[1: isCompressionEnabled=true] Thread=295, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1457 2024-11-20T21:21:53,763 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:53,770 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:53,771 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1732137713771 2024-11-20T21:21:53,812 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1732137713771 2024-11-20T21:21:53,817 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:53,820 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1732137713820 2024-11-20T21:21:53,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,832 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1732137713771 with entries=10, filesize=707 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1732137713820 2024-11-20T21:21:53,836 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:53,836 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1732137713771 is not closed yet, will try archiving it next time 2024-11-20T21:21:53,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741862_1038 (size=715) 2024-11-20T21:21:53,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741862_1038 (size=715) 2024-11-20T21:21:53,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741862_1038 (size=715) 2024-11-20T21:21:53,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741863_1039 (size=440) 2024-11-20T21:21:53,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741863_1039 (size=440) 2024-11-20T21:21:53,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741863_1039 (size=440) 2024-11-20T21:21:53,847 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=10, replicationBatchQueueCapacity=1 2024-11-20T21:21:53,849 DEBUG [Thread-540 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1732137713771, startPosition=0, beingWritten=false 2024-11-20T21:21:53,852 DEBUG [Thread-540 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1732137713771: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:53,856 DEBUG [Thread-540 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-20T21:21:53,856 DEBUG [Thread-540 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1732137713771 2024-11-20T21:21:53,856 DEBUG [Thread-540 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1732137713820, startPosition=0, beingWritten=false 2024-11-20T21:21:53,859 DEBUG [Thread-540 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1732137713820: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:53,862 DEBUG [Thread-540 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1732137713820 2024-11-20T21:21:53,862 DEBUG [Thread-540 {}] regionserver.ReplicationSourceWALReader(177): Read 5 WAL entries eligible for replication 2024-11-20T21:21:53,863 DEBUG [Thread-540 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:53,863 DEBUG [Thread-540 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:53,867 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:53,867 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137713820) 2024-11-20T21:21:53,880 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[1: isCompressionEnabled=true] Thread=300 (was 295) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #63] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1446 (was 1457) 2024-11-20T21:21:53,893 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[1: isCompressionEnabled=true] Thread=300, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1446 2024-11-20T21:21:53,893 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:53,897 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionForRecoveredQueue_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:53,898 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D.1732137713898 2024-11-20T21:21:53,911 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEOFExceptionForRecoveredQueue_1_ isCompressionEnabled=true]/testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D.1732137713898 2024-11-20T21:21:53,917 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:53,929 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-20T21:21:53,932 DEBUG [Thread-550 {}] regionserver.WALEntryStream(254): Creating new reader emptyLog, startPosition=0, beingWritten=false 2024-11-20T21:21:53,934 WARN [Thread-550 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: emptyLog, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-20T21:21:53,934 DEBUG [Thread-550 {}] regionserver.WALEntryStream(456): EOF, closing emptyLog 2024-11-20T21:21:53,934 DEBUG [Thread-550 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:53,934 DEBUG [Thread-550 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:53,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:53,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741864_1040 (size=93) 2024-11-20T21:21:53,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741864_1040 (size=93) 2024-11-20T21:21:53,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741864_1040 (size=93) 2024-11-20T21:21:53,947 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:53,947 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137713898) 2024-11-20T21:21:53,960 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[1: isCompressionEnabled=true] Thread=305 (was 300) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1439 (was 1446) 2024-11-20T21:21:53,973 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[1: isCompressionEnabled=true] Thread=305, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1439 2024-11-20T21:21:53,974 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:53,979 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:53,980 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713980 2024-11-20T21:21:53,994 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713980 2024-11-20T21:21:53,997 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:54,000 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713999 2024-11-20T21:21:54,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,019 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713980 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713999 2024-11-20T21:21:54,022 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:54,022 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713980 is not closed yet, will try archiving it next time 2024-11-20T21:21:54,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741865_1041 (size=212) 2024-11-20T21:21:54,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741865_1041 (size=212) 2024-11-20T21:21:54,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741865_1041 (size=212) 2024-11-20T21:21:54,027 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:54,030 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-20T21:21:54,031 DEBUG [Thread-567 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713980, startPosition=0, beingWritten=false 2024-11-20T21:21:54,035 DEBUG [Thread-567 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713980: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:54,037 DEBUG [Thread-567 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713980 2024-11-20T21:21:54,037 DEBUG [Thread-567 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-20T21:21:54,038 DEBUG [Thread-567 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713999, startPosition=0, beingWritten=true 2024-11-20T21:21:54,043 DEBUG [Thread-567 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713999: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:54,050 DEBUG [Thread-567 {}] regionserver.ReplicationSourceWALReader(177): Read 20 WAL entries eligible for replication 2024-11-20T21:21:54,051 DEBUG [Thread-567 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713999 to pos 1273, reset compression=false 2024-11-20T21:21:54,051 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051 2024-11-20T21:21:54,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,068 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713999 with entries=20, filesize=1.24 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051 2024-11-20T21:21:54,069 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:54,069 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713999 is not closed yet, will try archiving it next time 2024-11-20T21:21:54,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741866_1042 (size=1281) 2024-11-20T21:21:54,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741866_1042 (size=1281) 2024-11-20T21:21:54,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741866_1042 (size=1281) 2024-11-20T21:21:54,092 DEBUG [Thread-567 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713999 to pos 1273, reset compression=false 2024-11-20T21:21:54,096 DEBUG [Thread-567 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137713999 2024-11-20T21:21:54,096 DEBUG [Thread-567 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051, startPosition=0, beingWritten=true 2024-11-20T21:21:54,103 DEBUG [Thread-567 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:54,108 DEBUG [Thread-567 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-20T21:21:54,108 DEBUG [Thread-567 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051 to pos 715, reset compression=false 2024-11-20T21:21:54,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741867_1043 (size=723) 2024-11-20T21:21:54,116 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Sending block BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043] {}] datanode.DataXceiver(658): DatanodeRegistration(127.0.0.1:32873, datanodeUuid=8292258b-1538-4d48-b80d-4f0574e06a21, infoPort=35485, infoSecurePort=0, ipcPort=41015, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842):Got exception while serving BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043 to /127.0.0.1:50370 java.io.FileNotFoundException: BlockId 1073741867 is not valid. at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:810) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:801) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getMetaDataInputStream(FsDatasetImpl.java:246) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:319) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:21:54,116 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50370 [Sending block BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043] {}] datanode.DataXceiver(331): 127.0.0.1:32873:DataXceiver error processing READ_BLOCK operation src: /127.0.0.1:50370 dst: /127.0.0.1:32873 java.io.FileNotFoundException: BlockId 1073741867 is not valid. at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:810) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:801) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getMetaDataInputStream(FsDatasetImpl.java:246) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:319) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:21:54,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741867_1043 (size=723) 2024-11-20T21:21:54,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741867_1043 (size=723) 2024-11-20T21:21:54,120 WARN [Thread-567 {}] impl.BlockReaderFactory(772): I/O error constructing remote block reader for block BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043 java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:50812, remote=/127.0.0.1:32873, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051, for pool BP-1136921265-172.17.0.3-1732137701842 block 1073741867_1043 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-20T21:21:54,119 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50812 [Sending block BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043] {}] datanode.DataXceiver(658): DatanodeRegistration(127.0.0.1:32873, datanodeUuid=8292258b-1538-4d48-b80d-4f0574e06a21, infoPort=35485, infoSecurePort=0, ipcPort=41015, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842):Got exception while serving BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043 to /127.0.0.1:50812 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:21:54,120 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50812 [Sending block BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043] {}] datanode.DataXceiver(331): 127.0.0.1:32873:DataXceiver error processing READ_BLOCK operation src: /127.0.0.1:50812 dst: /127.0.0.1:32873 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:21:54,120 WARN [Thread-567 {}] hdfs.DFSInputStream(667): Failed to connect to /127.0.0.1:32873 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051 for block BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043, add to deadNodes and continue. java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:50812, remote=/127.0.0.1:32873, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051, for pool BP-1136921265-172.17.0.3-1732137701842 block 1073741867_1043 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-20T21:21:54,131 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:54,131 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137714051) 2024-11-20T21:21:54,146 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[1: isCompressionEnabled=true] Thread=310 (was 305) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #66] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-567 java.base@17.0.11/java.lang.ClassLoader.defineClass1(Native Method) java.base@17.0.11/java.lang.ClassLoader.defineClass(ClassLoader.java:1017) java.base@17.0.11/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:773) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:768) java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:318) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:781) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639) java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) java.base@17.0.11/java.lang.ClassLoader.loadClass(ClassLoader.java:525) app//org.apache.hbase.thirdparty.com.google.protobuf.DescriptorProtos$FileDescriptorProto.toBuilder(DescriptorProtos.java:2114) app//org.apache.hbase.thirdparty.com.google.protobuf.DescriptorProtos$FileDescriptorProto.newBuilder(DescriptorProtos.java:2107) app//org.apache.hbase.thirdparty.com.google.protobuf.DescriptorProtos$FileDescriptorProto$1.parsePartialFrom(DescriptorProtos.java:4485) app//org.apache.hbase.thirdparty.com.google.protobuf.DescriptorProtos$FileDescriptorProto$1.parsePartialFrom(DescriptorProtos.java:4479) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parsePartialFrom(AbstractParser.java:135) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:168) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:180) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:185) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) app//org.apache.hbase.thirdparty.com.google.protobuf.DescriptorProtos$FileDescriptorProto.parseFrom(DescriptorProtos.java:2056) app//org.apache.hbase.thirdparty.com.google.protobuf.Descriptors$FileDescriptor.internalBuildGeneratedFileFrom(Descriptors.java:466) app//org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.(HBaseProtos.java:27113) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.(WALProtos.java:15202) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.internalGetFieldAccessorTable(WALProtos.java:1640) app//org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.getDescriptorForType(GeneratedMessage.java:106) app//org.apache.hbase.thirdparty.com.google.protobuf.MessageReflection.findMissingFields(MessageReflection.java:151) app//org.apache.hbase.thirdparty.com.google.protobuf.MessageReflection.findMissingFields(MessageReflection.java:185) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage$Builder.newUninitializedMessageException(AbstractMessage.java:446) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage.newUninitializedMessageException(AbstractMessage.java:284) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.newUninitializedMessageException(AbstractParser.java:30) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:44) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) app//org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) - Thread LEAK? -, OpenFileDescriptor=425 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1512 (was 1439) - AvailableMemoryMB LEAK? - 2024-11-20T21:21:54,161 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[1: isCompressionEnabled=true] Thread=310, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1508 2024-11-20T21:21:54,162 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:54,169 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALKeySerialization_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:54,170 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D.1732137714170 2024-11-20T21:21:54,183 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALKeySerialization_1_ isCompressionEnabled=true]/testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D.1732137714170 2024-11-20T21:21:54,190 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:54,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741868_1044 (size=93) 2024-11-20T21:21:54,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741868_1044 (size=93) 2024-11-20T21:21:54,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741868_1044 (size=93) 2024-11-20T21:21:54,212 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:54,212 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137714170) 2024-11-20T21:21:54,231 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[1: isCompressionEnabled=true] Thread=315 (was 310) - Thread LEAK? -, OpenFileDescriptor=425 (was 425), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1482 (was 1508) 2024-11-20T21:21:54,246 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[1: isCompressionEnabled=true] Thread=315, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1480 2024-11-20T21:21:54,248 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:54,252 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:54,253 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1732137714253 2024-11-20T21:21:54,269 INFO [Thread-567 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=715, currentPosition=723 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-20T21:21:54,269 DEBUG [Thread-567 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051 for the last time to pos 715, reset compression=false 2024-11-20T21:21:54,272 INFO [Thread-567 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051 2024-11-20T21:21:54,279 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1732137714253 2024-11-20T21:21:54,281 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50828 [Sending block BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043] {}] datanode.DataXceiver(658): DatanodeRegistration(127.0.0.1:32873, datanodeUuid=8292258b-1538-4d48-b80d-4f0574e06a21, infoPort=35485, infoSecurePort=0, ipcPort=41015, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842):Got exception while serving BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043 to /127.0.0.1:50828 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:21:54,281 WARN [Thread-567 {}] impl.BlockReaderFactory(772): I/O error constructing remote block reader for block BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043 java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:50828, remote=/127.0.0.1:32873, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051, for pool BP-1136921265-172.17.0.3-1732137701842 block 1073741867_1043 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-20T21:21:54,281 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50828 [Sending block BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043] {}] datanode.DataXceiver(331): 127.0.0.1:32873:DataXceiver error processing READ_BLOCK operation src: /127.0.0.1:50828 dst: /127.0.0.1:32873 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:21:54,281 WARN [Thread-567 {}] hdfs.DFSInputStream(667): Failed to connect to /127.0.0.1:32873 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051 for block BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043, add to deadNodes and continue. java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1136921265-172.17.0.3-1732137701842:blk_1073741867_1043. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:50828, remote=/127.0.0.1:32873, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051, for pool BP-1136921265-172.17.0.3-1732137701842 block 1073741867_1043 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-20T21:21:54,284 INFO [Thread-567 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051 2024-11-20T21:21:54,285 DEBUG [Thread-567 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051 and length of the file is 723 2024-11-20T21:21:54,285 DEBUG [Thread-567 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137714051 2024-11-20T21:21:54,285 DEBUG [Thread-567 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:54,285 DEBUG [Thread-567 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:54,298 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:54,309 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1732137714253, startPosition=0, beingWritten=true 2024-11-20T21:21:54,323 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1732137714253: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:54,327 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-20T21:21:54,329 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:54,432 DEBUG [Thread-599 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1732137714253, startPosition=0, beingWritten=true 2024-11-20T21:21:54,438 DEBUG [Thread-599 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1732137714253: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:54,441 DEBUG [Thread-599 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-20T21:21:54,441 DEBUG [Thread-599 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1732137714253 to pos 314, reset compression=false 2024-11-20T21:21:54,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741869_1045 (size=322) 2024-11-20T21:21:54,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741869_1045 (size=322) 2024-11-20T21:21:54,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741869_1045 (size=322) 2024-11-20T21:21:54,459 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:54,459 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137714253) 2024-11-20T21:21:54,459 DEBUG [Thread-599 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1732137714253 2024-11-20T21:21:54,462 DEBUG [Thread-599 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:54,462 DEBUG [Thread-599 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:54,472 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[1: isCompressionEnabled=true] Thread=322 (was 315) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #69] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-599 java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) java.base@17.0.11/jdk.internal.reflect.GeneratedMethodAccessor29.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.mockito.internal.debugging.Java9PlusLocationImpl.stackWalk(Java9PlusLocationImpl.java:147) app//org.mockito.internal.debugging.Java9PlusLocationImpl.getStackFrame(Java9PlusLocationImpl.java:101) app//org.mockito.internal.debugging.Java9PlusLocationImpl.(Java9PlusLocationImpl.java:76) app//org.mockito.internal.debugging.LocationFactory$Java9PlusLocationFactory.create(LocationFactory.java:46) app//org.mockito.internal.debugging.LocationFactory.create(LocationFactory.java:19) app//org.mockito.internal.debugging.LocationFactory.create(LocationFactory.java:15) app//org.mockito.internal.creation.bytebuddy.MockMethodInterceptor.doIntercept(MockMethodInterceptor.java:56) app//org.mockito.internal.creation.bytebuddy.MockMethodInterceptor$DispatcherDefaultingToRealMethod.interceptSuperCallable(MockMethodInterceptor.java:145) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSource$MockitoMock$710IRJJn.getSourceManager(Unknown Source) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.getSourceManager(ReplicationSourceWALReader.java:437) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.checkBufferQuota(ReplicationSourceWALReader.java:274) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:145) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=425 (was 425), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1449 (was 1480) 2024-11-20T21:21:54,483 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[1: isCompressionEnabled=true] Thread=322, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1448 2024-11-20T21:21:54,484 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:54,487 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:54,488 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137714488 2024-11-20T21:21:54,497 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137714488 2024-11-20T21:21:54,505 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:54,516 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:54,516 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137714488, startPosition=0, beingWritten=true 2024-11-20T21:21:54,535 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137714488: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:54,540 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137714540 2024-11-20T21:21:54,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,563 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137714488 with entries=3, filesize=309 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137714540 2024-11-20T21:21:54,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741870_1046 (size=317) 2024-11-20T21:21:54,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741870_1046 (size=317) 2024-11-20T21:21:54,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741870_1046 (size=317) 2024-11-20T21:21:54,584 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:54,587 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:54,588 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:54,588 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137714488 for the last time to pos 253, reset compression=false 2024-11-20T21:21:54,596 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:54,596 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137714488 2024-11-20T21:21:54,696 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137714540, startPosition=0, beingWritten=true 2024-11-20T21:21:54,705 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1732137714540: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:54,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741871_1047 (size=205) 2024-11-20T21:21:54,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741871_1047 (size=205) 2024-11-20T21:21:54,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741871_1047 (size=205) 2024-11-20T21:21:54,744 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:54,744 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137714540) 2024-11-20T21:21:54,763 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[1: isCompressionEnabled=true] Thread=328 (was 322) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50848 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1404 (was 1448) 2024-11-20T21:21:54,778 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[1: isCompressionEnabled=true] Thread=328, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1401 2024-11-20T21:21:54,779 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:54,784 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:54,788 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1732137714785 2024-11-20T21:21:54,814 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1732137714785 2024-11-20T21:21:54,817 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:54,822 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:54,822 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1732137714785, startPosition=0, beingWritten=true 2024-11-20T21:21:54,829 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1732137714785: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:54,831 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1732137714830 2024-11-20T21:21:54,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,843 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1732137714785 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1732137714830 2024-11-20T21:21:54,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741872_1048 (size=212) 2024-11-20T21:21:54,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741872_1048 (size=212) 2024-11-20T21:21:54,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741872_1048 (size=212) 2024-11-20T21:21:54,848 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:54,848 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1732137714785 is not closed yet, will try archiving it next time 2024-11-20T21:21:54,850 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:54,850 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1732137714785 for the last time to pos 204, reset compression=false 2024-11-20T21:21:54,853 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1732137714785 and length of the file is 212 2024-11-20T21:21:54,853 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1732137714785 2024-11-20T21:21:54,953 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1732137714830, startPosition=0, beingWritten=true 2024-11-20T21:21:54,962 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1732137714830: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:54,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:54,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741873_1049 (size=212) 2024-11-20T21:21:54,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741873_1049 (size=212) 2024-11-20T21:21:54,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741873_1049 (size=212) 2024-11-20T21:21:54,972 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:54,972 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137714830) 2024-11-20T21:21:54,983 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[1: isCompressionEnabled=true] Thread=333 (was 328) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #70] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50848 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=752 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1391 (was 1401) 2024-11-20T21:21:54,994 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[1: isCompressionEnabled=true] Thread=333, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=752, ProcessCount=11, AvailableMemoryMB=1391 2024-11-20T21:21:54,995 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:54,998 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:54,999 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1732137714999 2024-11-20T21:21:55,011 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1732137714999 2024-11-20T21:21:55,015 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:55,019 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1732137714999, startPosition=0, beingWritten=true 2024-11-20T21:21:55,026 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1732137714999: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:55,028 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 225 and read up to 252 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:612) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALCellCodec$CompressedKvDecoder.parseCell(WALCellCodec.java:340) ~[classes/:?] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-20T21:21:56,029 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1732137714999 to pos 197, reset compression=true 2024-11-20T21:21:56,036 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 225 and read up to 252 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:615) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALCellCodec$CompressedKvDecoder.parseCell(WALCellCodec.java:340) ~[classes/:?] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-20T21:21:56,037 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1732137714999 to pos 197, reset compression=true 2024-11-20T21:21:56,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741874_1050 (size=261) 2024-11-20T21:21:56,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741874_1050 (size=261) 2024-11-20T21:21:56,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741874_1050 (size=261) 2024-11-20T21:21:56,056 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:56,056 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137714999) 2024-11-20T21:21:56,070 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[1: isCompressionEnabled=true] Thread=338 (was 333) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #72] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50848 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 752), ProcessCount=11 (was 11), AvailableMemoryMB=1452 (was 1391) - AvailableMemoryMB LEAK? - 2024-11-20T21:21:56,084 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[1: isCompressionEnabled=true] Thread=338, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=724, ProcessCount=11, AvailableMemoryMB=1452 2024-11-20T21:21:56,085 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:56,089 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:56,090 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137716090 2024-11-20T21:21:56,108 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137716090 2024-11-20T21:21:56,116 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:56,127 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137716090, startPosition=0, beingWritten=true 2024-11-20T21:21:56,147 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137716090: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:56,153 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137716090, startPosition=197, beingWritten=true 2024-11-20T21:21:56,175 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137716090: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:56,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741875_1051 (size=317) 2024-11-20T21:21:56,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741875_1051 (size=317) 2024-11-20T21:21:56,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741875_1051 (size=317) 2024-11-20T21:21:56,613 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:56,613 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137716090) 2024-11-20T21:21:56,623 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[1: isCompressionEnabled=true] Thread=342 (was 338) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #76] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 724), ProcessCount=11 (was 11), AvailableMemoryMB=1361 (was 1452) 2024-11-20T21:21:56,634 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[1: isCompressionEnabled=true] Thread=342, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=724, ProcessCount=11, AvailableMemoryMB=1360 2024-11-20T21:21:56,635 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:56,638 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:56,639 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639 2024-11-20T21:21:56,647 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639 2024-11-20T21:21:56,648 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:56,650 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639, startPosition=0, beingWritten=true 2024-11-20T21:21:56,656 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:56,658 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-20T21:21:56,659 DEBUG [Thread-669 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639, startPosition=0, beingWritten=true 2024-11-20T21:21:56,664 DEBUG [Thread-669 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:56,666 DEBUG [Thread-669 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-20T21:21:56,667 DEBUG [Thread-669 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639 to pos 314, reset compression=false 2024-11-20T21:21:56,673 DEBUG [Thread-669 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-20T21:21:56,673 DEBUG [Thread-669 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639 to pos 372, reset compression=false 2024-11-20T21:21:56,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741876_1052 (size=380) 2024-11-20T21:21:56,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741876_1052 (size=380) 2024-11-20T21:21:56,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741876_1052 (size=380) 2024-11-20T21:21:56,680 INFO [Thread-669 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=372, currentPosition=380 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-20T21:21:56,680 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:56,680 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137716639) 2024-11-20T21:21:56,680 DEBUG [Thread-669 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639 for the last time to pos 372, reset compression=false 2024-11-20T21:21:56,685 INFO [Thread-669 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639 2024-11-20T21:21:56,688 INFO [Thread-669 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639 2024-11-20T21:21:56,688 DEBUG [Thread-669 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639 and length of the file is 380 2024-11-20T21:21:56,688 DEBUG [Thread-669 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1732137716639 2024-11-20T21:21:56,689 DEBUG [Thread-669 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:56,689 DEBUG [Thread-669 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:56,691 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[1: isCompressionEnabled=true] Thread=348 (was 342) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Waiting for operation #80] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-669 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy48.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getBlockLocations$0(ClientNamenodeProtocolTranslatorPB.java:340) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$339/0x00007fbff8878fd0.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:340) jdk.internal.reflect.GeneratedMethodAccessor32.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy49.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:931) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:920) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:909) app//org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1076) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:342) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:338) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:355) app//org.apache.hadoop.fs.FileSystem.open(FileSystem.java:997) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50848 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 724), ProcessCount=11 (was 11), AvailableMemoryMB=1354 (was 1360) 2024-11-20T21:21:56,702 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[1: isCompressionEnabled=true] Thread=348, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=724, ProcessCount=11, AvailableMemoryMB=1352 2024-11-20T21:21:56,703 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:56,706 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:56,707 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 2024-11-20T21:21:56,715 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 2024-11-20T21:21:56,718 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:56,719 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716719 2024-11-20T21:21:56,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,727 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716719 2024-11-20T21:21:56,728 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:56,728 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 is not closed yet, will try archiving it next time 2024-11-20T21:21:56,729 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:56,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741877_1053 (size=93) 2024-11-20T21:21:56,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741877_1053 (size=93) 2024-11-20T21:21:56,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741877_1053 (size=93) 2024-11-20T21:21:56,732 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 2024-11-20T21:21:56,831 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 2024-11-20T21:21:56,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741879_1055 (size=85) 2024-11-20T21:21:56,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741879_1055 (size=85) 2024-11-20T21:21:56,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741879_1055 (size=85) 2024-11-20T21:21:56,848 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707, startPosition=0, beingWritten=false 2024-11-20T21:21:56,850 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 2024-11-20T21:21:56,852 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:56,853 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 for the last time to pos 0, reset compression=false 2024-11-20T21:21:56,854 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 2024-11-20T21:21:56,858 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 was moved to hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 2024-11-20T21:21:56,858 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 and length of the file is 85 2024-11-20T21:21:56,858 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716707 2024-11-20T21:21:56,858 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716719, startPosition=0, beingWritten=true 2024-11-20T21:21:56,863 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1732137716719: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:56,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741878_1054 (size=212) 2024-11-20T21:21:56,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741878_1054 (size=212) 2024-11-20T21:21:56,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741878_1054 (size=212) 2024-11-20T21:21:56,872 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:56,872 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137716719) 2024-11-20T21:21:56,883 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[1: isCompressionEnabled=true] Thread=353 (was 348) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #50] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 724), ProcessCount=11 (was 11), AvailableMemoryMB=1976 (was 1352) - AvailableMemoryMB LEAK? - 2024-11-20T21:21:56,893 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[1: isCompressionEnabled=true] Thread=353, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=724, ProcessCount=11, AvailableMemoryMB=2136 2024-11-20T21:21:56,894 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:56,897 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:56,898 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137716898 2024-11-20T21:21:56,906 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137716898 2024-11-20T21:21:56,913 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:56,919 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137716898, startPosition=0, beingWritten=true 2024-11-20T21:21:56,928 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137716898: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:56,932 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-20T21:21:56,933 DEBUG [Thread-707 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137716898, startPosition=0, beingWritten=true 2024-11-20T21:21:56,940 DEBUG [Thread-707 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137716898: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:56,943 DEBUG [Thread-707 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-20T21:21:56,944 DEBUG [Thread-707 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137716898 to pos 322, reset compression=false 2024-11-20T21:21:56,945 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:21:56,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:56,960 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Sending block BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056] {}] datanode.DataXceiver(658): DatanodeRegistration(127.0.0.1:37193, datanodeUuid=766fce34-fd71-4640-9e9d-b2afa05b747b, infoPort=41563, infoSecurePort=0, ipcPort=39029, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842):Got exception while serving BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056 to /127.0.0.1:47450 java.io.FileNotFoundException: BlockId 1073741880 is not valid. at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:810) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:801) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getMetaDataInputStream(FsDatasetImpl.java:246) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:319) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:21:56,961 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:47450 [Sending block BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056] {}] datanode.DataXceiver(331): 127.0.0.1:37193:DataXceiver error processing READ_BLOCK operation src: /127.0.0.1:47450 dst: /127.0.0.1:37193 java.io.FileNotFoundException: BlockId 1073741880 is not valid. at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:810) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:801) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getMetaDataInputStream(FsDatasetImpl.java:246) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:319) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:21:56,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741880_1056 (size=330) 2024-11-20T21:21:56,964 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:53100 [Sending block BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056] {}] datanode.DataXceiver(658): DatanodeRegistration(127.0.0.1:37193, datanodeUuid=766fce34-fd71-4640-9e9d-b2afa05b747b, infoPort=41563, infoSecurePort=0, ipcPort=39029, storageInfo=lv=-57;cid=testClusterID;nsid=2116147380;c=1732137701842):Got exception while serving BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056 to /127.0.0.1:53100 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:21:56,964 WARN [Thread-707 {}] impl.BlockReaderFactory(772): I/O error constructing remote block reader for block BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056 java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:53100, remote=/127.0.0.1:37193, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137716898, for pool BP-1136921265-172.17.0.3-1732137701842 block 1073741880_1056 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-20T21:21:56,964 WARN [Thread-707 {}] hdfs.DFSInputStream(667): Failed to connect to /127.0.0.1:37193 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137716898 for block BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056, add to deadNodes and continue. java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:53100, remote=/127.0.0.1:37193, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137716898, for pool BP-1136921265-172.17.0.3-1732137701842 block 1073741880_1056 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-20T21:21:56,964 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:53100 [Sending block BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056] {}] datanode.DataXceiver(331): 127.0.0.1:37193:DataXceiver error processing READ_BLOCK operation src: /127.0.0.1:53100 dst: /127.0.0.1:37193 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-1136921265-172.17.0.3-1732137701842:blk_1073741880_1056. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:21:56,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741880_1056 (size=330) 2024-11-20T21:21:56,967 DEBUG [Thread-707 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137716898 for the last time to pos 322, reset compression=false 2024-11-20T21:21:56,971 DEBUG [Thread-707 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137716898 and length of the file is 330 2024-11-20T21:21:56,971 DEBUG [Thread-707 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1732137716898 2024-11-20T21:21:56,971 DEBUG [Thread-707 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-20T21:21:56,971 DEBUG [Thread-707 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-20T21:21:56,971 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:56,972 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137716898) 2024-11-20T21:21:56,990 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[1: isCompressionEnabled=true] Thread=359 (was 353) Potentially hanging thread: Thread-707 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:343) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.replicationDone(ReplicationSourceWALReader.java:123) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:151) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #51] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50848 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=425 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 724), ProcessCount=11 (was 11), AvailableMemoryMB=3366 (was 2136) - AvailableMemoryMB LEAK? - 2024-11-20T21:21:57,006 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[1: isCompressionEnabled=true] Thread=359, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=724, ProcessCount=9, AvailableMemoryMB=3435 2024-11-20T21:21:57,007 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:57,011 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEmptyStream_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:57,012 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1732137717012 2024-11-20T21:21:57,034 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1732137717012 2024-11-20T21:21:57,048 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097)] 2024-11-20T21:21:57,063 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1732137717012, startPosition=0, beingWritten=true 2024-11-20T21:21:57,073 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1732137717012: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:57,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741881_1057 (size=93) 2024-11-20T21:21:57,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741881_1057 (size=93) 2024-11-20T21:21:57,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741881_1057 (size=93) 2024-11-20T21:21:57,111 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:57,111 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137717012) 2024-11-20T21:21:57,128 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[1: isCompressionEnabled=true] Thread=364 (was 359) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #53] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=425 (was 425), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 724), ProcessCount=11 (was 9) - ProcessCount LEAK? -, AvailableMemoryMB=3390 (was 3435) 2024-11-20T21:21:57,153 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[1: isCompressionEnabled=true] Thread=364, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=724, ProcessCount=11, AvailableMemoryMB=3387 2024-11-20T21:21:57,154 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:57,164 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:57,166 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717165 2024-11-20T21:21:57,214 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717165 2024-11-20T21:21:57,229 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:57,246 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717165, startPosition=0, beingWritten=true 2024-11-20T21:21:57,258 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717165: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:57,262 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:57,262 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717165, startPosition=204, beingWritten=true 2024-11-20T21:21:57,274 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717165: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:57,278 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717278 2024-11-20T21:21:57,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,288 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717165 with entries=3, filesize=330 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717278 2024-11-20T21:21:57,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741882_1058 (size=338) 2024-11-20T21:21:57,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741882_1058 (size=338) 2024-11-20T21:21:57,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741882_1058 (size=338) 2024-11-20T21:21:57,291 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:57,291 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717165 is not closed yet, will try archiving it next time 2024-11-20T21:21:57,294 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:57,294 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717165, startPosition=267, beingWritten=false 2024-11-20T21:21:57,324 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717165: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:57,325 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:57,325 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717165 2024-11-20T21:21:57,426 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717278, startPosition=0, beingWritten=true 2024-11-20T21:21:57,433 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1732137717278: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:57,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741883_1059 (size=212) 2024-11-20T21:21:57,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741883_1059 (size=212) 2024-11-20T21:21:57,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741883_1059 (size=212) 2024-11-20T21:21:57,451 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:57,451 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137717278) 2024-11-20T21:21:57,465 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[1: isCompressionEnabled=true] Thread=370 (was 364) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:53134 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50848 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 724), ProcessCount=11 (was 11), AvailableMemoryMB=3280 (was 3387) 2024-11-20T21:21:57,484 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[1: isCompressionEnabled=true] Thread=370, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=724, ProcessCount=11, AvailableMemoryMB=3276 2024-11-20T21:21:57,485 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:57,496 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:57,497 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137717497 2024-11-20T21:21:57,525 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137717497 2024-11-20T21:21:57,541 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:57,569 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137717497, startPosition=0, beingWritten=true 2024-11-20T21:21:57,595 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137717497: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:57,602 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137717497, startPosition=204, beingWritten=true 2024-11-20T21:21:57,629 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1732137717497: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:57,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:57,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741884_1060 (size=330) 2024-11-20T21:21:57,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741884_1060 (size=330) 2024-11-20T21:21:57,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741884_1060 (size=330) 2024-11-20T21:21:58,089 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:58,089 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137717497) 2024-11-20T21:21:58,101 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[1: isCompressionEnabled=true] Thread=375 (was 370) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:35222 [Waiting for operation #57] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 724), ProcessCount=11 (was 11), AvailableMemoryMB=3256 (was 3276) 2024-11-20T21:21:58,118 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[1: isCompressionEnabled=true] Thread=375, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=724, ProcessCount=11, AvailableMemoryMB=3248 2024-11-20T21:21:58,119 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-20T21:21:58,129 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs, maxLogs=32 2024-11-20T21:21:58,130 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1732137718130 2024-11-20T21:21:58,142 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1732137718130 2024-11-20T21:21:58,149 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35485:35485),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:41563:41563)] 2024-11-20T21:21:58,157 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1732137718157 2024-11-20T21:21:58,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:58,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:58,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:58,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:58,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:58,185 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1732137718130 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1732137718157 2024-11-20T21:21:58,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741885_1061 (size=212) 2024-11-20T21:21:58,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741885_1061 (size=212) 2024-11-20T21:21:58,189 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41563:41563),(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35485:35485)] 2024-11-20T21:21:58,190 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1732137718130 is not closed yet, will try archiving it next time 2024-11-20T21:21:58,190 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:21:58,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741885_1061 (size=212) 2024-11-20T21:21:58,296 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1732137718130, startPosition=0, beingWritten=false 2024-11-20T21:21:58,301 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1732137718130: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T21:21:58,302 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:43273/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1732137718130 2024-11-20T21:21:58,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:58,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:58,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:58,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:58,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:21:58,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741886_1062 (size=93) 2024-11-20T21:21:58,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43731 is added to blk_1073741886_1062 (size=93) 2024-11-20T21:21:58,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37193 is added to blk_1073741886_1062 (size=93) 2024-11-20T21:21:58,317 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/oldWALs 2024-11-20T21:21:58,318 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D:(num 1732137718157) 2024-11-20T21:21:58,332 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[1: isCompressionEnabled=true] Thread=380 (was 375) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434732769_22 at /127.0.0.1:50848 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 724), ProcessCount=11 (was 11), AvailableMemoryMB=3215 (was 3248) 2024-11-20T21:21:58,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T21:21:58,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6d954ade{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:21:58,351 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@76890781{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T21:21:58,351 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T21:21:58,351 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dfa9f68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T21:21:58,352 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f179af2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/hadoop.log.dir/,STOPPED} 2024-11-20T21:21:58,362 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T21:21:58,362 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T21:21:58,363 WARN [BP-1136921265-172.17.0.3-1732137701842 heartbeating to localhost/127.0.0.1:43273 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T21:21:58,363 WARN [BP-1136921265-172.17.0.3-1732137701842 heartbeating to localhost/127.0.0.1:43273 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1136921265-172.17.0.3-1732137701842 (Datanode Uuid ac7636a0-3b69-421b-8050-d02f66429448) service to localhost/127.0.0.1:43273 2024-11-20T21:21:58,365 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/cluster_284eea24-a7a8-0acf-c040-e490a965681e/data/data5/current/BP-1136921265-172.17.0.3-1732137701842 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:21:58,366 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/cluster_284eea24-a7a8-0acf-c040-e490a965681e/data/data6/current/BP-1136921265-172.17.0.3-1732137701842 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:21:58,367 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T21:21:58,395 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@fecc8a3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:21:58,395 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@cbbb804{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T21:21:58,395 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T21:21:58,396 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45d510a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T21:21:58,396 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cd634bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/hadoop.log.dir/,STOPPED} 2024-11-20T21:21:58,398 WARN [BP-1136921265-172.17.0.3-1732137701842 heartbeating to localhost/127.0.0.1:43273 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T21:21:58,398 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T21:21:58,398 WARN [BP-1136921265-172.17.0.3-1732137701842 heartbeating to localhost/127.0.0.1:43273 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1136921265-172.17.0.3-1732137701842 (Datanode Uuid 8292258b-1538-4d48-b80d-4f0574e06a21) service to localhost/127.0.0.1:43273 2024-11-20T21:21:58,398 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T21:21:58,399 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/cluster_284eea24-a7a8-0acf-c040-e490a965681e/data/data3/current/BP-1136921265-172.17.0.3-1732137701842 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:21:58,399 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/cluster_284eea24-a7a8-0acf-c040-e490a965681e/data/data4/current/BP-1136921265-172.17.0.3-1732137701842 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:21:58,399 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T21:21:58,412 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5a86aa33{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:21:58,412 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@53cf346{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T21:21:58,412 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T21:21:58,412 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d2a62e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T21:21:58,413 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@309d3e97{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/hadoop.log.dir/,STOPPED} 2024-11-20T21:21:58,414 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T21:21:58,414 WARN [BP-1136921265-172.17.0.3-1732137701842 heartbeating to localhost/127.0.0.1:43273 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T21:21:58,415 WARN [BP-1136921265-172.17.0.3-1732137701842 heartbeating to localhost/127.0.0.1:43273 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1136921265-172.17.0.3-1732137701842 (Datanode Uuid 766fce34-fd71-4640-9e9d-b2afa05b747b) service to localhost/127.0.0.1:43273 2024-11-20T21:21:58,415 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T21:21:58,416 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/cluster_284eea24-a7a8-0acf-c040-e490a965681e/data/data1/current/BP-1136921265-172.17.0.3-1732137701842 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:21:58,416 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/cluster_284eea24-a7a8-0acf-c040-e490a965681e/data/data2/current/BP-1136921265-172.17.0.3-1732137701842 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:21:58,417 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T21:21:58,428 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@44567ea4{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T21:21:58,429 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@216da5e5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T21:21:58,429 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T21:21:58,429 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a9b22e2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T21:21:58,430 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c2d1933{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c827147-5f5c-96d0-ad21-77d7b2729355/hadoop.log.dir/,STOPPED} 2024-11-20T21:21:58,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down